Examples with GridQueryTypeDescriptor org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor used on opensource projects

Example 21 with GridQueryTypeDescriptor

use of org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor in project ignite by apache.

the class IndexQueryReducer method pageComparator.

/**
 * {@inheritDoc}
 */
@Override
protected CompletableFuture<Comparator<NodePage<R>>> pageComparator() {
    return metaFut.thenApply(m -> {
        LinkedHashMap<String, IndexKeyDefinition> keyDefs = m.keyDefinitions();
        GridQueryTypeDescriptor typeDesc = cctx.kernalContext().query().typeDescriptor(cctx.name(), QueryUtils.typeName(valType));
        return new IndexedNodePageComparator(m, typeDesc, keyDefs);
    });
}

Example 22 with GridQueryTypeDescriptor

use of org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor in project ignite by apache.

the class GridCacheQuerySqlMetadataJobV2 method call.

/**
 * {@inheritDoc}
 */
@Override
public Collection<GridCacheQueryManager.CacheSqlMetadata> call() {
    final GridKernalContext ctx = ((IgniteKernal) ignite).context();
    Collection<String> cacheNames = F.viewReadOnly(ctx.cache().caches(), new C1<IgniteInternalCache<?, ?>, String>() {

        @Override
        public String apply(IgniteInternalCache<?, ?> c) {
            return c.name();
        }
    }, new P1<IgniteInternalCache<?, ?>>() {

        @Override
        public boolean apply(IgniteInternalCache<?, ?> c) {
            return !CU.isSystemCache(c.name()) && !DataStructuresProcessor.isDataStructureCache(c.name());
        }
    });
    return F.transform(cacheNames, new C1<String, GridCacheQueryManager.CacheSqlMetadata>() {

        @Override
        public GridCacheQueryManager.CacheSqlMetadata apply(String cacheName) {
            Collection<GridQueryTypeDescriptor> types = ctx.query().types(cacheName);
            Collection<String> names = U.newHashSet(types.size());
            Map<String, String> keyClasses = U.newHashMap(types.size());
            Map<String, String> valClasses = U.newHashMap(types.size());
            Map<String, Map<String, String>> fields = U.newHashMap(types.size());
            Map<String, Collection<GridCacheSqlIndexMetadata>> indexes = U.newHashMap(types.size());
            Map<String, Set<String>> notNullFields = U.newHashMap(types.size());
            for (GridQueryTypeDescriptor type : types) {
                // Filter internal types (e.g., data structures).
                if (type.name().startsWith("GridCache"))
                    continue;
                names.add(type.name());
                keyClasses.put(type.name(), type.keyClass().getName());
                valClasses.put(type.name(), type.valueClass().getName());
                int size = type.fields().isEmpty() ? NO_FIELDS_COLUMNS_COUNT : type.fields().size();
                Map<String, String> fieldsMap = U.newLinkedHashMap(size);
                HashSet<String> notNullFieldsSet = U.newHashSet(1);
                // _KEY and _VAL are not included in GridIndexingTypeDescriptor.valueFields
                if (type.fields().isEmpty()) {
                    fieldsMap.put("_KEY", type.keyClass().getName());
                    fieldsMap.put("_VAL", type.valueClass().getName());
                }
                for (Map.Entry<String, Class<?>> e : type.fields().entrySet()) {
                    String fieldName = e.getKey();
                    fieldsMap.put(fieldName.toUpperCase(), e.getValue().getName());
                    if (type.property(fieldName).notNull())
                        notNullFieldsSet.add(fieldName.toUpperCase());
                }
                fields.put(type.name(), fieldsMap);
                notNullFields.put(type.name(), notNullFieldsSet);
                Map<String, GridQueryIndexDescriptor> idxs = type.indexes();
                Collection<GridCacheSqlIndexMetadata> indexesCol = new ArrayList<>(idxs.size());
                for (Map.Entry<String, GridQueryIndexDescriptor> e : idxs.entrySet()) {
                    GridQueryIndexDescriptor desc = e.getValue();
                    // Add only SQL indexes.
                    if (desc.type() == QueryIndexType.SORTED) {
                        Collection<String> idxFields = new LinkedList<>();
                        Collection<String> descendings = new LinkedList<>();
                        for (String idxField : e.getValue().fields()) {
                            String idxFieldUpper = idxField.toUpperCase();
                            idxFields.add(idxFieldUpper);
                            if (desc.descending(idxField))
                                descendings.add(idxFieldUpper);
                        }
                        indexesCol.add(new GridCacheQueryManager.CacheSqlIndexMetadata(e.getKey().toUpperCase(), idxFields, descendings, false));
                    }
                }
                indexes.put(type.name(), indexesCol);
            }
            return new GridCacheQuerySqlMetadataV2(cacheName, names, keyClasses, valClasses, fields, indexes, notNullFields);
        }
    });
}

Example 23 with GridQueryTypeDescriptor

use of org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor in project ignite by apache.

the class UpdatePlanBuilder method planForInsert.

/**
 * Prepare update plan for INSERT or MERGE.
 *
 * @param planKey Plan key.
 * @param stmt INSERT or MERGE statement.
 * @param idx Indexing.
 * @param mvccEnabled Mvcc flag.
 * @return Update plan.
 * @throws IgniteCheckedException if failed.
 */
@SuppressWarnings("ConstantConditions")
private static UpdatePlan planForInsert(QueryDescriptor planKey, GridSqlStatement stmt, IgniteH2Indexing idx, boolean mvccEnabled, IgniteLogger log, boolean forceFillAbsentPKsWithDefaults) throws IgniteCheckedException {
    GridSqlQuery sel = null;
    GridSqlElement target;
    GridSqlColumn[] cols;
    boolean isTwoStepSubqry;
    int rowsNum;
    GridSqlTable tbl;
    GridH2RowDescriptor desc;
    List<GridSqlElement[]> elRows = null;
    UpdateMode mode;
    if (stmt instanceof GridSqlInsert) {
        mode = UpdateMode.INSERT;
        GridSqlInsert ins = (GridSqlInsert) stmt;
        target = ins.into();
        tbl = DmlAstUtils.gridTableForElement(target);
        GridH2Table h2Tbl = tbl.dataTable();
        assert h2Tbl != null;
        desc = h2Tbl.rowDescriptor();
        cols = ins.columns();
        if (noQuery(ins.rows()))
            elRows = ins.rows();
        else
            sel = DmlAstUtils.selectForInsertOrMerge(cols, ins.rows(), ins.query());
        isTwoStepSubqry = (ins.query() != null);
        rowsNum = isTwoStepSubqry ? 0 : ins.rows().size();
    } else if (stmt instanceof GridSqlMerge) {
        mode = UpdateMode.MERGE;
        GridSqlMerge merge = (GridSqlMerge) stmt;
        target = merge.into();
        tbl = DmlAstUtils.gridTableForElement(target);
        desc = tbl.dataTable().rowDescriptor();
        cols = merge.columns();
        if (noQuery(merge.rows()))
            elRows = merge.rows();
        else
            sel = DmlAstUtils.selectForInsertOrMerge(cols, merge.rows(), merge.query());
        isTwoStepSubqry = (merge.query() != null);
        rowsNum = isTwoStepSubqry ? 0 : merge.rows().size();
    } else {
        throw new IgniteSQLException("Unexpected DML operation [cls=" + stmt.getClass().getName() + ']', IgniteQueryErrorCode.UNEXPECTED_OPERATION);
    }
    // Let's set the flag only for subqueries that have their FROM specified.
    isTwoStepSubqry &= (sel != null && (sel instanceof GridSqlUnion || (sel instanceof GridSqlSelect && ((GridSqlSelect) sel).from() != null)));
    int keyColIdx = -1;
    int valColIdx = -1;
    boolean hasKeyProps = false;
    boolean hasValProps = false;
    if (desc == null)
        throw new IgniteSQLException("Row descriptor undefined for table '" + tbl.dataTable().getName() + "'", IgniteQueryErrorCode.NULL_TABLE_DESCRIPTOR);
    GridCacheContext<?, ?> cctx = desc.context();
    String[] colNames = new String[cols.length];
    int[] colTypes = new int[cols.length];
    GridQueryTypeDescriptor type = desc.type();
    Set<String> rowKeys = desc.getRowKeyColumnNames();
    boolean onlyVisibleColumns = true;
    for (int i = 0; i < cols.length; i++) {
        GridSqlColumn col = cols[i];
        if (!col.column().getVisible())
            onlyVisibleColumns = false;
        String colName = col.columnName();
        colNames[i] = colName;
        colTypes[i] = col.resultType().type();
        rowKeys.remove(colName);
        int colId = col.column().getColumnId();
        if (desc.isKeyColumn(colId)) {
            keyColIdx = i;
            continue;
        }
        if (desc.isValueColumn(colId)) {
            valColIdx = i;
            continue;
        }
        GridQueryProperty prop = desc.type().property(colName);
        assert prop != null : "Property '" + colName + "' not found.";
        if (prop.key())
            hasKeyProps = true;
        else
            hasValProps = true;
    }
    rowKeys.removeIf(rowKey -> desc.type().property(rowKey).defaultValue() != null);
    boolean fillAbsentPKsWithNullsOrDefaults = type.fillAbsentPKsWithDefaults() || forceFillAbsentPKsWithDefaults;
    if (fillAbsentPKsWithNullsOrDefaults && onlyVisibleColumns && !rowKeys.isEmpty()) {
        String[] extendedColNames = new String[rowKeys.size() + colNames.length];
        int[] extendedColTypes = new int[rowKeys.size() + colTypes.length];
        System.arraycopy(colNames, 0, extendedColNames, 0, colNames.length);
        System.arraycopy(colTypes, 0, extendedColTypes, 0, colTypes.length);
        int currId = colNames.length;
        for (String key : rowKeys) {
            Column col = tbl.dataTable().getColumn(key);
            extendedColNames[currId] = col.getName();
            extendedColTypes[currId] = col.getType();
            currId++;
        }
        colNames = extendedColNames;
        colTypes = extendedColTypes;
    }
    verifyDmlColumns(tbl.dataTable(), F.viewReadOnly(Arrays.asList(cols), TO_H2_COL));
    KeyValueSupplier keySupplier = createSupplier(cctx, desc.type(), keyColIdx, hasKeyProps, true, false);
    KeyValueSupplier valSupplier = createSupplier(cctx, desc.type(), valColIdx, hasValProps, false, false);
    String selectSql = sel != null ? sel.getSQL() : null;
    DmlDistributedPlanInfo distributed = null;
    if (rowsNum == 0 && !F.isEmpty(selectSql)) {
        distributed = checkPlanCanBeDistributed(idx, mvccEnabled, planKey, selectSql, tbl.dataTable().cacheName(), log);
    }
    List<List<DmlArgument>> rows = null;
    if (elRows != null) {
        assert sel == null;
        rows = new ArrayList<>(elRows.size());
        for (GridSqlElement[] elRow : elRows) {
            List<DmlArgument> row = new ArrayList<>(cols.length);
            for (GridSqlElement el : elRow) {
                DmlArgument arg = DmlArguments.create(el);
                row.add(arg);
            }
            rows.add(row);
        }
    }
    return new UpdatePlan(mode, tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, selectSql, !isTwoStepSubqry, rows, rowsNum, null, distributed, false, fillAbsentPKsWithNullsOrDefaults);
}

Example 24 with GridQueryTypeDescriptor

use of org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor in project ignite by apache.

the class QueryIndexDefinition method treeName.

/**
 * {@inheritDoc}
 */
@Override
public String treeName() {
    GridH2RowDescriptor rowDesc = table.rowDescriptor();
    String typeIdStr = "";
    if (rowDesc != null) {
        GridQueryTypeDescriptor typeDesc = rowDesc.type();
        int typeId = cctx.binaryMarshaller() ? typeDesc.typeId() : typeDesc.valueClass().hashCode();
        typeIdStr = typeId + "_";
    }
    // Legacy in treeName from H2Tree.
    return BPlusTree.treeName(typeIdStr + idxName().idxName(), "H2Tree");
}

Example 25 with GridQueryTypeDescriptor

use of org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor in project ignite by apache.

the class ValidateIndexesClosure method call0.

/**
 */
private VisorValidateIndexesJobResult call0() {
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    Set<Integer> grpIds = collectGroupIds();
    /**
     * Update counters per partition per group.
     */
    final Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrp = getUpdateCountersSnapshot(ignite, grpIds);
    IdleVerifyUtility.IdleChecker idleChecker = new IdleVerifyUtility.IdleChecker(ignite, partsWithCntrsPerGrp);
    List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
    List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
    totalCacheGrps = grpIds.size();
    Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds, idleChecker);
    GridQueryProcessor qryProcessor = ignite.context().query();
    IgniteH2Indexing h2Indexing = (IgniteH2Indexing) qryProcessor.getIndexing();
    for (Integer grpId : grpIds) {
        CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
        if (isNull(grpCtx) || integrityCheckResults.containsKey(grpId))
            continue;
        for (GridDhtLocalPartition part : grpCtx.topology().localPartitions()) partArgs.add(new T2<>(grpCtx, part));
        for (GridCacheContext ctx : grpCtx.caches()) {
            String cacheName = ctx.name();
            if (cacheNames == null || cacheNames.contains(cacheName)) {
                Collection<GridQueryTypeDescriptor> types = qryProcessor.types(cacheName);
                if (F.isEmpty(types))
                    continue;
                for (GridQueryTypeDescriptor type : types) {
                    GridH2Table gridH2Tbl = h2Indexing.schemaManager().dataTable(cacheName, type.tableName());
                    if (isNull(gridH2Tbl))
                        continue;
                    for (Index idx : gridH2Tbl.getIndexes()) {
                        if (idx instanceof H2TreeIndexBase)
                            idxArgs.add(new T2<>(ctx, idx));
                    }
                }
            }
        }
    }
    // To decrease contention on same indexes.
    shuffle(partArgs);
    shuffle(idxArgs);
    totalPartitions = partArgs.size();
    totalIndexes = idxArgs.size();
    List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>(partArgs.size());
    List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>(idxArgs.size());
    List<T3<CacheGroupContext, GridDhtLocalPartition, Future<CacheSize>>> cacheSizeFutures = new ArrayList<>(partArgs.size());
    List<T3<GridCacheContext, Index, Future<T2<Throwable, Long>>>> idxSizeFutures = new ArrayList<>(idxArgs.size());
    partArgs.forEach(k -> procPartFutures.add(processPartitionAsync(k.get1(), k.get2())));
    idxArgs.forEach(k -> procIdxFutures.add(processIndexAsync(k, idleChecker)));
    if (checkSizes) {
        for (T2<CacheGroupContext, GridDhtLocalPartition> partArg : partArgs) {
            CacheGroupContext cacheGrpCtx = partArg.get1();
            GridDhtLocalPartition locPart = partArg.get2();
            cacheSizeFutures.add(new T3<>(cacheGrpCtx, locPart, calcCacheSizeAsync(cacheGrpCtx, locPart)));
        }
        for (T2<GridCacheContext, Index> idxArg : idxArgs) {
            GridCacheContext cacheCtx = idxArg.get1();
            Index idx = idxArg.get2();
            idxSizeFutures.add(new T3<>(cacheCtx, idx, calcIndexSizeAsync(cacheCtx, idx, idleChecker)));
        }
    }
    Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
    Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
    Map<String, ValidateIndexesCheckSizeResult> checkSizeResults = new HashMap<>();
    int curPart = 0;
    int curIdx = 0;
    int curCacheSize = 0;
    int curIdxSize = 0;
    try {
        for (; curPart < procPartFutures.size(); curPart++) {
            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
            if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                partResults.putAll(partRes);
        }
        for (; curIdx < procIdxFutures.size(); curIdx++) {
            Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
            Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
            if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
                idxResults.putAll(idxRes);
        }
        if (checkSizes) {
            for (; curCacheSize < cacheSizeFutures.size(); curCacheSize++) cacheSizeFutures.get(curCacheSize).get3().get();
            for (; curIdxSize < idxSizeFutures.size(); curIdxSize++) idxSizeFutures.get(curIdxSize).get3().get();
            checkSizes(cacheSizeFutures, idxSizeFutures, checkSizeResults);
            Map<Integer, Map<Integer, PartitionUpdateCounter>> partsWithCntrsPerGrpAfterChecks = getUpdateCountersSnapshot(ignite, grpIds);
            List<Integer> diff = compareUpdateCounters(ignite, partsWithCntrsPerGrp, partsWithCntrsPerGrpAfterChecks);
            if (!F.isEmpty(diff)) {
                String res = formatUpdateCountersDiff(ignite, diff);
                if (!res.isEmpty())
                    throw new GridNotIdleException(GRID_NOT_IDLE_MSG + "[" + res + "]");
            }
        }
        log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and " + totalIndexes + " indexes.");
    } catch (InterruptedException | ExecutionException e) {
        for (int j = curPart; j < procPartFutures.size(); j++) procPartFutures.get(j).cancel(false);
        for (int j = curIdx; j < procIdxFutures.size(); j++) procIdxFutures.get(j).cancel(false);
        for (int j = curCacheSize; j < cacheSizeFutures.size(); j++) cacheSizeFutures.get(j).get3().cancel(false);
        for (int j = curIdxSize; j < idxSizeFutures.size(); j++) idxSizeFutures.get(j).get3().cancel(false);
        throw unwrapFutureException(e);
    }
    if (validateCtx.isCancelled())
        throw new IgniteException(CANCELLED_MSG);
    return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values(), checkSizeResults);
}