Examples with SessionState org.apache.hadoop.hive.ql.session.SessionState used on opensource projects

Example 66 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class KeyVerifyingStatsAggregator method aggregateStats.

@Override
public String aggregateStats(String keyPrefix, String statType) {
    SessionState ss = SessionState.get();
    // Have to use the length instead of the actual prefix because the prefix is location dependent
    // 17 is 16 (16 byte MD5 hash) + 1 for the path separator
    // Can be less than 17 due to unicode characters
    ss.out.println("Stats prefix is hashed: " + new Boolean(keyPrefix.length() <= 17));
    return null;
}

Example 67 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class EnforceReadOnlyTables method run.

@Override
public void run(HookContext hookContext) throws Exception {
    SessionState ss = SessionState.get();
    Set<ReadEntity> inputs = hookContext.getInputs();
    Set<WriteEntity> outputs = hookContext.getOutputs();
    UserGroupInformation ugi = hookContext.getUgi();
    this.run(ss, inputs, outputs, ugi);
}

Example 68 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class Driver method acquireLocksAndOpenTxn.

/**
   * Acquire read and write locks needed by the statement. The list of objects to be locked are
   * obtained from the inputs and outputs populated by the compiler. The lock acquisition scheme is
   * pretty simple. If all the locks cannot be obtained, error out. Deadlock is avoided by making
   * sure that the locks are lexicographically sorted.
   *
   * This method also records the list of valid transactions.  This must be done after any
   * transactions have been opened and locks acquired.
   * @param startTxnImplicitly in AC=false, the 1st DML starts a txn
   **/
private int acquireLocksAndOpenTxn(boolean startTxnImplicitly) {
    PerfLogger perfLogger = SessionState.getPerfLogger();
    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
    SessionState ss = SessionState.get();
    HiveTxnManager txnMgr = ss.getTxnMgr();
    if (startTxnImplicitly) {
        assert !txnMgr.getAutoCommit();
    }
    try {
        // Don't use the userName member, as it may or may not have been set.  Get the value from
        // conf, which calls into getUGI to figure out who the process is running as.
        String userFromUGI;
        try {
            userFromUGI = conf.getUser();
        } catch (IOException e) {
            errorMessage = "FAILED: Error in determining user while acquiring locks: " + e.getMessage();
            SQLState = ErrorMsg.findSQLState(e.getMessage());
            downstreamError = e;
            console.printError(errorMessage, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
            return 10;
        }
        boolean initiatingTransaction = false;
        boolean readOnlyQueryInAutoCommit = false;
        if ((txnMgr.getAutoCommit() && haveAcidWrite()) || plan.getOperation() == HiveOperation.START_TRANSACTION || (!txnMgr.getAutoCommit() && startTxnImplicitly)) {
            if (txnMgr.isTxnOpen()) {
                throw new RuntimeException("Already have an open transaction txnid:" + txnMgr.getCurrentTxnId());
            }
            // We are writing to tables in an ACID compliant way, so we need to open a transaction
            txnMgr.openTxn(ctx, userFromUGI);
            initiatingTransaction = true;
        } else {
            readOnlyQueryInAutoCommit = txnMgr.getAutoCommit() && plan.getOperation() == HiveOperation.QUERY && !haveAcidWrite();
        }
        // Set the transaction id in all of the acid file sinks
        if (haveAcidWrite()) {
            for (FileSinkDesc desc : acidSinks) {
                desc.setTransactionId(txnMgr.getCurrentTxnId());
                //it's possible to have > 1 FileSink writing to the same table/partition
                //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
                desc.setStatementId(txnMgr.getWriteIdAndIncrement());
            }
        }
        /*Note, we have to record snapshot after lock acquisition to prevent lost update problem
      consider 2 concurrent "update table T set x = x + 1".  1st will get the locks and the
      2nd will block until 1st one commits and only then lock in the snapshot, i.e. it will
      see the changes made by 1st one.  This takes care of autoCommit=true case.
      For multi-stmt txns this is not sufficient and will be managed via WriteSet tracking
      in the lock manager.*/
        txnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState);
        if (initiatingTransaction || (readOnlyQueryInAutoCommit && acidInQuery)) {
            //For multi-stmt txns we should record the snapshot when txn starts but
            // don't update it after that until txn completes.  Thus the check for {@code initiatingTransaction}
            //For autoCommit=true, Read-only statements, txn is implicit, i.e. lock in the snapshot
            //for each statement.
            recordValidTxns();
        }
        return 0;
    } catch (Exception e) {
        errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
        SQLState = ErrorMsg.findSQLState(e.getMessage());
        downstreamError = e;
        console.printError(errorMessage, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
        return 10;
    } finally {
        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
    }
}

Example 69 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class Hive method createIndex.

/**
   *
   * @param tableName
   *          table name
   * @param indexName
   *          index name
   * @param indexHandlerClass
   *          index handler class
   * @param indexedCols
   *          index columns
   * @param indexTblName
   *          index table's name
   * @param deferredRebuild
   *          referred build index table's data
   * @param inputFormat
   *          input format
   * @param outputFormat
   *          output format
   * @param serde
   * @param storageHandler
   *          index table's storage handler
   * @param location
   *          location
   * @param idxProps
   *          idx
   * @param serdeProps
   *          serde properties
   * @param collItemDelim
   * @param fieldDelim
   * @param fieldEscape
   * @param lineDelim
   * @param mapKeyDelim
   * @throws HiveException
   */
public void createIndex(String tableName, String indexName, String indexHandlerClass, List<String> indexedCols, String indexTblName, boolean deferredRebuild, String inputFormat, String outputFormat, String serde, String storageHandler, String location, Map<String, String> idxProps, Map<String, String> tblProps, Map<String, String> serdeProps, String collItemDelim, String fieldDelim, String fieldEscape, String lineDelim, String mapKeyDelim, String indexComment) throws HiveException {
    try {
        String tdname = Utilities.getDatabaseName(tableName);
        String idname = Utilities.getDatabaseName(indexTblName);
        if (!idname.equals(tdname)) {
            throw new HiveException("Index on different database (" + idname + ") from base table (" + tdname + ") is not supported.");
        }
        Index old_index = null;
        try {
            old_index = getIndex(tableName, indexName);
        } catch (Exception e) {
        }
        if (old_index != null) {
            throw new HiveException("Index " + indexName + " already exists on table " + tableName);
        }
        org.apache.hadoop.hive.metastore.api.Table baseTbl = getTable(tableName).getTTable();
        if (TableType.VIRTUAL_VIEW.toString().equals(baseTbl.getTableType())) {
            throw new HiveException("tableName=" + tableName + " is a VIRTUAL VIEW. Index on VIRTUAL VIEW is not supported.");
        }
        if (baseTbl.isTemporary()) {
            throw new HiveException("tableName=" + tableName + " is a TEMPORARY TABLE. Index on TEMPORARY TABLE is not supported.");
        }
        org.apache.hadoop.hive.metastore.api.Table temp = null;
        try {
            temp = getTable(indexTblName).getTTable();
        } catch (Exception e) {
        }
        if (temp != null) {
            throw new HiveException("Table name " + indexTblName + " already exists. Choose another name.");
        }
        SerDeInfo serdeInfo = new SerDeInfo();
        serdeInfo.setName(indexTblName);
        if (serde != null) {
            serdeInfo.setSerializationLib(serde);
        } else {
            if (storageHandler == null) {
                serdeInfo.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
            } else {
                HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler);
                String serDeClassName = sh.getSerDeClass().getName();
                serdeInfo.setSerializationLib(serDeClassName);
            }
        }
        serdeInfo.setParameters(new HashMap<String, String>());
        if (fieldDelim != null) {
            serdeInfo.getParameters().put(FIELD_DELIM, fieldDelim);
            serdeInfo.getParameters().put(SERIALIZATION_FORMAT, fieldDelim);
        }
        if (fieldEscape != null) {
            serdeInfo.getParameters().put(ESCAPE_CHAR, fieldEscape);
        }
        if (collItemDelim != null) {
            serdeInfo.getParameters().put(COLLECTION_DELIM, collItemDelim);
        }
        if (mapKeyDelim != null) {
            serdeInfo.getParameters().put(MAPKEY_DELIM, mapKeyDelim);
        }
        if (lineDelim != null) {
            serdeInfo.getParameters().put(LINE_DELIM, lineDelim);
        }
        if (serdeProps != null) {
            Iterator<Entry<String, String>> iter = serdeProps.entrySet().iterator();
            while (iter.hasNext()) {
                Entry<String, String> m = iter.next();
                serdeInfo.getParameters().put(m.getKey(), m.getValue());
            }
        }
        List<FieldSchema> indexTblCols = new ArrayList<FieldSchema>();
        List<Order> sortCols = new ArrayList<Order>();
        int k = 0;
        Table metaBaseTbl = new Table(baseTbl);
        // Even though we are storing these in metastore, get regular columns. Indexes on lengthy
        // types from e.g. Avro schema will just fail to create the index table (by design).
        List<FieldSchema> cols = metaBaseTbl.getCols();
        for (int i = 0; i < cols.size(); i++) {
            FieldSchema col = cols.get(i);
            if (indexedCols.contains(col.getName())) {
                indexTblCols.add(col);
                sortCols.add(new Order(col.getName(), 1));
                k++;
            }
        }
        if (k != indexedCols.size()) {
            throw new RuntimeException("Check the index columns, they should appear in the table being indexed.");
        }
        int time = (int) (System.currentTimeMillis() / 1000);
        org.apache.hadoop.hive.metastore.api.Table tt = null;
        HiveIndexHandler indexHandler = HiveUtils.getIndexHandler(this.getConf(), indexHandlerClass);
        String itname = Utilities.getTableName(indexTblName);
        if (indexHandler.usesIndexTable()) {
            tt = new org.apache.hadoop.hive.ql.metadata.Table(idname, itname).getTTable();
            List<FieldSchema> partKeys = baseTbl.getPartitionKeys();
            tt.setPartitionKeys(partKeys);
            tt.setTableType(TableType.INDEX_TABLE.toString());
            if (tblProps != null) {
                for (Entry<String, String> prop : tblProps.entrySet()) {
                    tt.putToParameters(prop.getKey(), prop.getValue());
                }
            }
            SessionState ss = SessionState.get();
            CreateTableAutomaticGrant grants;
            if (ss != null && ((grants = ss.getCreateTableGrants()) != null)) {
                PrincipalPrivilegeSet principalPrivs = new PrincipalPrivilegeSet();
                principalPrivs.setUserPrivileges(grants.getUserGrants());
                principalPrivs.setGroupPrivileges(grants.getGroupGrants());
                principalPrivs.setRolePrivileges(grants.getRoleGrants());
                tt.setPrivileges(principalPrivs);
            }
        }
        if (!deferredRebuild) {
            throw new RuntimeException("Please specify deferred rebuild using \" WITH DEFERRED REBUILD \".");
        }
        StorageDescriptor indexSd = new StorageDescriptor(indexTblCols, location, inputFormat, outputFormat, false, /*compressed - not used*/
        -1, /*numBuckets - default is -1 when the table has no buckets*/
        serdeInfo, null, /*bucketCols*/
        sortCols, null);
        String ttname = Utilities.getTableName(tableName);
        Index indexDesc = new Index(indexName, indexHandlerClass, tdname, ttname, time, time, itname, indexSd, new HashMap<String, String>(), deferredRebuild);
        if (indexComment != null) {
            indexDesc.getParameters().put("comment", indexComment);
        }
        if (idxProps != null) {
            indexDesc.getParameters().putAll(idxProps);
        }
        indexHandler.analyzeIndexDefinition(baseTbl, indexDesc, tt);
        this.getMSC().createIndex(indexDesc, tt);
    } catch (Exception e) {
        throw new HiveException(e);
    }
}

Example 70 with SessionState

use of org.apache.hadoop.hive.ql.session.SessionState in project hive by apache.

the class Hive method createTable.

/**
   * Creates the table with the given objects. It takes additional arguments for
   * primary keys and foreign keys associated with the table.
   *
   * @param tbl
   *          a table object
   * @param ifNotExists
   *          if true, ignore AlreadyExistsException
   * @param primaryKeys
   *          primary key columns associated with the table
   * @param foreignKeys
   *          foreign key columns associated with the table
   * @throws HiveException
   */
public void createTable(Table tbl, boolean ifNotExists, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys) throws HiveException {
    try {
        if (tbl.getDbName() == null || "".equals(tbl.getDbName().trim())) {
            tbl.setDbName(SessionState.get().getCurrentDatabase());
        }
        if (tbl.getCols().size() == 0 || tbl.getSd().getColsSize() == 0) {
            tbl.setFields(MetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), tbl.getDeserializer()));
        }
        tbl.checkValidity(conf);
        if (tbl.getParameters() != null) {
            tbl.getParameters().remove(hive_metastoreConstants.DDL_TIME);
        }
        org.apache.hadoop.hive.metastore.api.Table tTbl = tbl.getTTable();
        PrincipalPrivilegeSet principalPrivs = new PrincipalPrivilegeSet();
        SessionState ss = SessionState.get();
        if (ss != null) {
            CreateTableAutomaticGrant grants = ss.getCreateTableGrants();
            if (grants != null) {
                principalPrivs.setUserPrivileges(grants.getUserGrants());
                principalPrivs.setGroupPrivileges(grants.getGroupGrants());
                principalPrivs.setRolePrivileges(grants.getRoleGrants());
                tTbl.setPrivileges(principalPrivs);
            }
        }
        if (primaryKeys == null && foreignKeys == null) {
            getMSC().createTable(tTbl);
        } else {
            getMSC().createTableWithConstraints(tTbl, primaryKeys, foreignKeys);
        }
    } catch (AlreadyExistsException e) {
        if (!ifNotExists) {
            throw new HiveException(e);
        }
    } catch (Exception e) {
        throw new HiveException(e);
    }
}