Examples with Collection java.util.Collection used on opensource projects

Example 91 with Collection

use of java.util.Collection in project hive by apache.

the class ObjectStore method getForeignKeysViaJdo.

private List<SQLForeignKey> getForeignKeysViaJdo(String parent_db_name, String parent_tbl_name, String foreign_db_name, String foreign_tbl_name) throws MetaException {
    boolean commited = false;
    List<SQLForeignKey> foreignKeys = null;
    Collection<?> constraints = null;
    Query query = null;
    Map<String, String> tblToConstraint = new HashMap<String, String>();
    try {
        openTransaction();
        String queryText = (parent_tbl_name != null ? "parentTable.tableName == parent_tbl_name && " : "") + (parent_db_name != null ? " parentTable.database.name == parent_db_name && " : "") + (foreign_tbl_name != null ? " childTable.tableName == foreign_tbl_name && " : "") + (foreign_db_name != null ? " childTable.database.name == foreign_db_name && " : "") + " constraintType == MConstraint.FOREIGN_KEY_CONSTRAINT";
        queryText = queryText.trim();
        query = pm.newQuery(MConstraint.class, queryText);
        String paramText = (parent_tbl_name == null ? "" : "java.lang.String parent_tbl_name,") + (parent_db_name == null ? "" : " java.lang.String parent_db_name, ") + (foreign_tbl_name == null ? "" : "java.lang.String foreign_tbl_name,") + (foreign_db_name == null ? "" : " java.lang.String foreign_db_name");
        paramText = paramText.trim();
        if (paramText.endsWith(",")) {
            paramText = paramText.substring(0, paramText.length() - 1);
        }
        query.declareParameters(paramText);
        List<String> params = new ArrayList<String>();
        if (parent_tbl_name != null) {
            params.add(parent_tbl_name);
        }
        if (parent_db_name != null) {
            params.add(parent_db_name);
        }
        if (foreign_tbl_name != null) {
            params.add(foreign_tbl_name);
        }
        if (foreign_db_name != null) {
            params.add(foreign_db_name);
        }
        if (params.size() == 0) {
            constraints = (Collection<?>) query.execute();
        } else if (params.size() == 1) {
            constraints = (Collection<?>) query.execute(params.get(0));
        } else if (params.size() == 2) {
            constraints = (Collection<?>) query.execute(params.get(0), params.get(1));
        } else if (params.size() == 3) {
            constraints = (Collection<?>) query.execute(params.get(0), params.get(1), params.get(2));
        } else {
            constraints = (Collection<?>) query.executeWithArray(params.get(0), params.get(1), params.get(2), params.get(3));
        }
        pm.retrieveAll(constraints);
        foreignKeys = new ArrayList<SQLForeignKey>();
        for (Iterator<?> i = constraints.iterator(); i.hasNext(); ) {
            MConstraint currPKFK = (MConstraint) i.next();
            int enableValidateRely = currPKFK.getEnableValidateRely();
            boolean enable = (enableValidateRely & 4) != 0;
            boolean validate = (enableValidateRely & 2) != 0;
            boolean rely = (enableValidateRely & 1) != 0;
            String consolidatedtblName = currPKFK.getParentTable().getDatabase().getName() + "." + currPKFK.getParentTable().getTableName();
            String pkName;
            if (tblToConstraint.containsKey(consolidatedtblName)) {
                pkName = tblToConstraint.get(consolidatedtblName);
            } else {
                pkName = getPrimaryKeyConstraintName(currPKFK.getParentTable().getDatabase().getName(), currPKFK.getParentTable().getDatabase().getName());
                tblToConstraint.put(consolidatedtblName, pkName);
            }
            foreignKeys.add(new SQLForeignKey(currPKFK.getParentTable().getDatabase().getName(), currPKFK.getParentTable().getDatabase().getName(), currPKFK.getParentColumn().getCols().get(currPKFK.getParentIntegerIndex()).getName(), currPKFK.getChildTable().getDatabase().getName(), currPKFK.getChildTable().getTableName(), currPKFK.getChildColumn().getCols().get(currPKFK.getChildIntegerIndex()).getName(), currPKFK.getPosition(), currPKFK.getUpdateRule(), currPKFK.getDeleteRule(), currPKFK.getConstraintName(), pkName, enable, validate, rely));
        }
        commited = commitTransaction();
    } finally {
        if (!commited) {
            rollbackTransaction();
        }
        if (query != null) {
            query.closeAll();
        }
    }
    return foreignKeys;
}

Example 92 with Collection

use of java.util.Collection in project hive by apache.

the class ObjectStore method listAllTableConstraintsWithOptionalConstraintName.

private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName(String dbName, String tableName, String constraintname) {
    List<MConstraint> mConstraints = null;
    List<String> constraintNames = new ArrayList<String>();
    Query query = null;
    try {
        query = pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint  where " + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname) || " + "(childTable != null && childTable.tableName == ctblname && " + "childTable.database.name == cdbname)) " + (constraintname != null ? " && constraintName == constraintname" : ""));
        query.declareParameters("java.lang.String ptblname, java.lang.String pdbname," + "java.lang.String ctblname, java.lang.String cdbname" + (constraintname != null ? ", java.lang.String constraintname" : ""));
        Collection<?> constraintNamesColl = constraintname != null ? ((Collection<?>) query.executeWithArray(tableName, dbName, tableName, dbName, constraintname)) : ((Collection<?>) query.executeWithArray(tableName, dbName, tableName, dbName));
        for (Iterator<?> i = constraintNamesColl.iterator(); i.hasNext(); ) {
            String currName = (String) i.next();
            constraintNames.add(currName);
        }
        query = pm.newQuery(MConstraint.class);
        query.setFilter("param.contains(constraintName)");
        query.declareParameters("java.util.Collection param");
        Collection<?> constraints = (Collection<?>) query.execute(constraintNames);
        mConstraints = new ArrayList<MConstraint>();
        for (Iterator<?> i = constraints.iterator(); i.hasNext(); ) {
            MConstraint currConstraint = (MConstraint) i.next();
            mConstraints.add(currConstraint);
        }
    } finally {
        if (query != null) {
            query.closeAll();
        }
    }
    return mConstraints;
}

Example 93 with Collection

use of java.util.Collection in project hive by apache.

the class ObjectStore method listPartitionNamesByFilter.

@Override
public List<String> listPartitionNamesByFilter(String dbName, String tableName, String filter, short maxParts) throws MetaException {
    boolean success = false;
    Query query = null;
    List<String> partNames = new ArrayList<String>();
    try {
        openTransaction();
        LOG.debug("Executing listMPartitionNamesByFilter");
        dbName = HiveStringUtils.normalizeIdentifier(dbName);
        tableName = HiveStringUtils.normalizeIdentifier(tableName);
        MTable mtable = getMTable(dbName, tableName);
        if (mtable == null) {
            // table or db does not exist, we return an empty list
            return partNames;
        }
        Map<String, Object> params = new HashMap<String, Object>();
        String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params);
        query = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + "where " + queryFilterString);
        if (maxParts >= 0) {
            // User specified a row limit, set it on the Query
            query.setRange(0, maxParts);
        }
        LOG.debug("Filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString);
        LOG.debug("Parms is " + params);
        String parameterDeclaration = makeParameterDeclarationStringObj(params);
        query.declareParameters(parameterDeclaration);
        query.setOrdering("partitionName ascending");
        query.setResult("partitionName");
        Collection names = (Collection) query.executeWithMap(params);
        partNames = new ArrayList<String>();
        for (Iterator i = names.iterator(); i.hasNext(); ) {
            partNames.add((String) i.next());
        }
        LOG.debug("Done executing query for listMPartitionNamesByFilter");
        success = commitTransaction();
        LOG.debug("Done retrieving all objects for listMPartitionNamesByFilter");
    } finally {
        if (!success) {
            rollbackTransaction();
        }
        if (query != null) {
            query.closeAll();
        }
    }
    return partNames;
}

Example 94 with Collection

use of java.util.Collection in project hive by apache.

the class CustomPartitionVertex method processAllEvents.

private void processAllEvents(String inputName, Multimap<Integer, InputSplit> bucketToGroupedSplitMap, boolean secondLevelGroupingDone) throws IOException {
    int totalInputsCount = 0;
    List<Integer> numSplitsForTask = new ArrayList<Integer>();
    for (Entry<Integer, Collection<InputSplit>> entry : bucketToGroupedSplitMap.asMap().entrySet()) {
        int bucketNum = entry.getKey();
        Collection<InputSplit> initialSplits = entry.getValue();
        finalSplits.addAll(initialSplits);
        for (InputSplit inputSplit : initialSplits) {
            bucketToTaskMap.put(bucketNum, taskCount);
            if (secondLevelGroupingDone) {
                TezGroupedSplit groupedSplit = (TezGroupedSplit) inputSplit;
                numSplitsForTask.add(groupedSplit.getGroupedSplits().size());
                totalInputsCount += groupedSplit.getGroupedSplits().size();
            } else {
                numSplitsForTask.add(1);
                totalInputsCount += 1;
            }
            taskCount++;
        }
    }
    inputNameInputSpecMap.put(inputName, InputSpecUpdate.createPerTaskInputSpecUpdate(numSplitsForTask));
    // Construct the EdgeManager descriptor to be used by all edges which need
    // the routing table.
    EdgeManagerPluginDescriptor hiveEdgeManagerDesc = null;
    if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) {
        hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName());
        UserPayload payload = getBytePayload(bucketToTaskMap);
        hiveEdgeManagerDesc.setUserPayload(payload);
    }
    // Replace the edge manager for all vertices which have routing type custom.
    for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) {
        if (edgeEntry.getValue().getDataMovementType() == DataMovementType.CUSTOM && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName().equals(CustomPartitionEdge.class.getName())) {
            emMap.put(edgeEntry.getKey(), hiveEdgeManagerDesc);
        }
    }
    LOG.info("Task count is " + taskCount + " for input name: " + inputName);
    List<InputDataInformationEvent> taskEvents = Lists.newArrayListWithCapacity(totalInputsCount);
    // Re-serialize the splits after grouping.
    int count = 0;
    for (InputSplit inputSplit : finalSplits) {
        if (secondLevelGroupingDone) {
            TezGroupedSplit tezGroupedSplit = (TezGroupedSplit) inputSplit;
            for (InputSplit subSplit : tezGroupedSplit.getGroupedSplits()) {
                if ((subSplit instanceof TezGroupedSplit) == false) {
                    throw new IOException("Unexpected split type found: " + subSplit.getClass().getCanonicalName());
                }
                MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(subSplit);
                InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit.toByteString().asReadOnlyByteBuffer());
                diEvent.setTargetIndex(count);
                taskEvents.add(diEvent);
            }
        } else {
            MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit);
            InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit.toByteString().asReadOnlyByteBuffer());
            diEvent.setTargetIndex(count);
            taskEvents.add(diEvent);
        }
        count++;
    }
    // Set the actual events for the tasks.
    LOG.info("For input name: " + inputName + " task events size is " + taskEvents.size());
    context.addRootInputEvents(inputName, taskEvents);
    if (inputToGroupedSplitMap.isEmpty() == false) {
        for (Entry<String, Multimap<Integer, InputSplit>> entry : inputToGroupedSplitMap.entrySet()) {
            processAllSideEvents(entry.getKey(), entry.getValue());
        }
        setVertexParallelismAndRootInputSpec(inputNameInputSpecMap);
        inputToGroupedSplitMap.clear();
    }
    // Only done when it is a bucket map join only no SMB.
    if (numInputsAffectingRootInputSpecUpdate == 1) {
        setVertexParallelismAndRootInputSpec(inputNameInputSpecMap);
    }
}

Example 95 with Collection

use of java.util.Collection in project hive by apache.

the class SparkUtilities method rddToString.

private static void rddToString(RDD rdd, StringBuilder sb, String offset) {
    sb.append(offset).append(rdd.getClass().getCanonicalName()).append("[").append(rdd.hashCode()).append("]");
    if (rdd.getStorageLevel().useMemory()) {
        sb.append("(cached)");
    }
    sb.append("\n");
    Collection<Dependency> dependencies = JavaConversions.asJavaCollection(rdd.dependencies());
    if (dependencies != null) {
        offset += "\t";
        for (Dependency dependency : dependencies) {
            RDD parentRdd = dependency.rdd();
            rddToString(parentRdd, sb, offset);
        }
    } else if (rdd instanceof UnionRDD) {
        UnionRDD unionRDD = (UnionRDD) rdd;
        offset += "\t";
        Collection<RDD> parentRdds = JavaConversions.asJavaCollection(unionRDD.rdds());
        for (RDD parentRdd : parentRdds) {
            rddToString(parentRdd, sb, offset);
        }
    }
}