Examples with OProfiler com.orientechnologies.common.profiler.OProfiler used on opensource projects

Example 1 with OProfiler

use of com.orientechnologies.common.profiler.OProfiler in project orientdb by orientechnologies.

the class OCommandCacheSoftRefs method get.

@Override
public Object get(final OSecurityUser iUser, final String queryText, final int iLimit) {
    if (!enable)
        return null;
    OCachedResult result;
    synchronized (this) {
        final String key = getKey(iUser, queryText, iLimit);
        result = cache.get(key);
        if (result != null) {
            // SERIALIZE ALL THE RECORDS IN LOCK TO AVOID CONCURRENT ACCESS. ONCE SERIALIZED CAN ARE THREAD-SAFE
            int resultsetSize = 1;
            if (result.result instanceof ORecord)
                ((ORecord) result.result).toStream();
            else if (OMultiValue.isMultiValue(result.result)) {
                resultsetSize = OMultiValue.getSize(result.result);
                for (Object rc : OMultiValue.getMultiValueIterable(result.result)) {
                    if (rc != null && rc instanceof ORecord) {
                        ((ORecord) rc).toStream();
                    }
                }
            }
            if (OLogManager.instance().isDebugEnabled())
                OLogManager.instance().debug(this, "Reused cached resultset size=%d", resultsetSize);
        }
    }
    final OProfiler profiler = Orient.instance().getProfiler();
    if (profiler.isRecording()) {
        // UPDATE PROFILER
        if (result != null) {
            profiler.updateCounter(profiler.getDatabaseMetric(databaseName, "queryCache.hit"), "Results returned by Query Cache", +1);
        } else {
            profiler.updateCounter(profiler.getDatabaseMetric(databaseName, "queryCache.miss"), "Results not returned by Query Cache", +1);
        }
    }
    return result != null ? result.result : null;
}

Example 2 with OProfiler

use of com.orientechnologies.common.profiler.OProfiler in project orientdb by orientechnologies.

the class OQueryOperator method updateProfiler.

protected void updateProfiler(final OCommandContext iContext, final OIndex<?> index, final List<Object> keyParams, final OIndexDefinition indexDefinition) {
    if (iContext.isRecordingMetrics())
        iContext.updateMetric("compositeIndexUsed", +1);
    final OProfiler profiler = Orient.instance().getProfiler();
    if (profiler.isRecording()) {
        profiler.updateCounter(profiler.getDatabaseMetric(index.getDatabaseName(), "query.indexUsed"), "Used index in query", +1);
        int params = indexDefinition.getParamCount();
        if (params > 1) {
            final String profiler_prefix = profiler.getDatabaseMetric(index.getDatabaseName(), "query.compositeIndexUsed");
            profiler.updateCounter(profiler_prefix, "Used composite index in query", +1);
            profiler.updateCounter(profiler_prefix + "." + params, "Used composite index in query with " + params + " params", +1);
            profiler.updateCounter(profiler_prefix + "." + params + '.' + keyParams.size(), "Used composite index in query with " + params + " params and " + keyParams.size() + " keys", +1);
        }
    }
}

Example 3 with OProfiler

use of com.orientechnologies.common.profiler.OProfiler in project orientdb by orientechnologies.

the class PolymorphicQueryTest method testSubclassesIndexes.

@Test
public void testSubclassesIndexes() throws Exception {
    database.begin();
    OProfiler profiler = Orient.instance().getProfiler();
    long indexUsage = profiler.getCounter("db.demo.query.indexUsed");
    long indexUsageReverted = profiler.getCounter("db.demo.query.indexUseAttemptedAndReverted");
    if (indexUsage < 0) {
        indexUsage = 0;
    }
    if (indexUsageReverted < 0) {
        indexUsageReverted = 0;
    }
    profiler.startRecording();
    for (int i = 0; i < 10000; i++) {
        final ODocument doc1 = new ODocument("IndexInSubclassesTestChild1");
        doc1.field("name", "name" + i);
        doc1.save();
        final ODocument doc2 = new ODocument("IndexInSubclassesTestChild2");
        doc2.field("name", "name" + i);
        doc2.save();
        if (i % 100 == 0) {
            database.commit();
        }
    }
    database.commit();
    List<ODocument> result = database.query(new OSQLSynchQuery<ODocument>("select from IndexInSubclassesTestBase where name > 'name9995' and name < 'name9999' order by name ASC"));
    Assert.assertEquals(result.size(), 6);
    String lastName = result.get(0).field("name");
    for (int i = 1; i < result.size(); i++) {
        ODocument current = result.get(i);
        String currentName = current.field("name");
        Assert.assertTrue(lastName.compareTo(currentName) <= 0);
        lastName = currentName;
    }
    Assert.assertEquals(profiler.getCounter("db.demo.query.indexUsed"), indexUsage + 2);
    long reverted = profiler.getCounter("db.demo.query.indexUseAttemptedAndReverted");
    Assert.assertEquals(reverted < 0 ? 0 : reverted, indexUsageReverted);
    result = database.query(new OSQLSynchQuery<ODocument>("select from IndexInSubclassesTestBase where name > 'name9995' and name < 'name9999' order by name DESC"));
    Assert.assertEquals(result.size(), 6);
    lastName = result.get(0).field("name");
    for (int i = 1; i < result.size(); i++) {
        ODocument current = result.get(i);
        String currentName = current.field("name");
        Assert.assertTrue(lastName.compareTo(currentName) >= 0);
        lastName = currentName;
    }
    profiler.stopRecording();
}

Example 4 with OProfiler

use of com.orientechnologies.common.profiler.OProfiler in project orientdb by orientechnologies.

the class OChainedIndexProxy method updateStatistic.

/**
   * Register statistic information about usage of index in {@link OProfilerStub}.
   * 
   * @param index
   *          which usage is registering.
   */
private void updateStatistic(OIndex<?> index) {
    final OProfiler profiler = Orient.instance().getProfiler();
    if (profiler.isRecording()) {
        Orient.instance().getProfiler().updateCounter(profiler.getDatabaseMetric(index.getDatabaseName(), "query.indexUsed"), "Used index in query", +1);
        final int paramCount = index.getDefinition().getParamCount();
        if (paramCount > 1) {
            final String profiler_prefix = profiler.getDatabaseMetric(index.getDatabaseName(), "query.compositeIndexUsed");
            profiler.updateCounter(profiler_prefix, "Used composite index in query", +1);
            profiler.updateCounter(profiler_prefix + "." + paramCount, "Used composite index in query with " + paramCount + " params", +1);
        }
    }
}

Example 5 with OProfiler

use of com.orientechnologies.common.profiler.OProfiler in project orientdb by orientechnologies.

the class OCommandExecutorSQLSelect method searchForIndexes.

@SuppressWarnings("rawtypes")
private boolean searchForIndexes(final OClass iSchemaClass) {
    if (uniqueResult != null)
        uniqueResult.clear();
    final ODatabaseDocument database = getDatabase();
    database.checkSecurity(ORule.ResourceGeneric.CLASS, ORole.PERMISSION_READ, iSchemaClass.getName().toLowerCase());
    // fetch all possible variants of subqueries that can be used in indexes.
    if (compiledFilter == null) {
        return tryOptimizeSort(iSchemaClass);
    }
    // try indexed functions
    Iterator<OIdentifiable> fetchedFromFunction = tryIndexedFunctions(iSchemaClass);
    if (fetchedFromFunction != null) {
        fetchFromTarget(fetchedFromFunction);
        return true;
    }
    // the main condition is a set of sub-conditions separated by OR operators
    final List<List<OIndexSearchResult>> conditionHierarchy = filterAnalyzer.analyzeMainCondition(compiledFilter.getRootCondition(), iSchemaClass, context);
    if (conditionHierarchy == null)
        return false;
    List<OIndexCursor> cursors = new ArrayList<OIndexCursor>();
    boolean indexIsUsedInOrderBy = false;
    List<IndexUsageLog> indexUseAttempts = new ArrayList<IndexUsageLog>();
    try {
        //to track if the index used is specific for this class or if it's defined on a super/sub class
        boolean indexOnExactClass = true;
        OIndexSearchResult lastSearchResult = null;
        for (List<OIndexSearchResult> indexSearchResults : conditionHierarchy) {
            // go through all variants to choose which one can be used for index search.
            boolean indexUsed = false;
            for (final OIndexSearchResult searchResult : indexSearchResults) {
                lastSearchResult = searchResult;
                final List<OIndex<?>> involvedIndexes = filterAnalyzer.getInvolvedIndexes(iSchemaClass, searchResult);
                Collections.sort(involvedIndexes, new IndexComparator());
                indexOnExactClass = true;
                // go through all possible index for given set of fields.
                for (final OIndex index : involvedIndexes) {
                    final long indexRebuildVersion = index.getRebuildVersion();
                    if (index.isRebuilding()) {
                        continue;
                    }
                    final OIndexDefinition indexDefinition = index.getDefinition();
                    if (searchResult.containsNullValues && indexDefinition.isNullValuesIgnored()) {
                        continue;
                    }
                    final OQueryOperator operator = searchResult.lastOperator;
                    // are equals.
                    if (!OIndexSearchResult.isIndexEqualityOperator(operator)) {
                        final String lastFiled = searchResult.lastField.getItemName(searchResult.lastField.getItemCount() - 1);
                        final String relatedIndexField = indexDefinition.getFields().get(searchResult.fieldValuePairs.size());
                        if (!lastFiled.equals(relatedIndexField)) {
                            continue;
                        }
                    }
                    final int searchResultFieldsCount = searchResult.fields().size();
                    final List<Object> keyParams = new ArrayList<Object>(searchResultFieldsCount);
                    // We get only subset contained in processed sub query.
                    for (final String fieldName : indexDefinition.getFields().subList(0, searchResultFieldsCount)) {
                        Object fieldValue = searchResult.fieldValuePairs.get(fieldName);
                        if (fieldValue instanceof OSQLQuery<?> || fieldValue instanceof OSQLFilterCondition) {
                            return false;
                        }
                        if (fieldValue != null) {
                            keyParams.add(fieldValue);
                        } else {
                            if (searchResult.lastValue instanceof OSQLQuery<?> || searchResult.lastValue instanceof OSQLFilterCondition) {
                                return false;
                            }
                            keyParams.add(searchResult.lastValue);
                        }
                    }
                    metricRecorder.recordInvolvedIndexesMetric(index);
                    OIndexCursor cursor;
                    indexIsUsedInOrderBy = orderByOptimizer.canBeUsedByOrderByAfterFilter(index, getEqualsClausesPrefix(searchResult), orderedFields) && !(index.getInternal() instanceof OChainedIndexProxy);
                    try {
                        boolean ascSortOrder = !indexIsUsedInOrderBy || orderedFields.get(0).getValue().equals(KEYWORD_ASC);
                        if (indexIsUsedInOrderBy) {
                            fullySortedByIndex = expandTarget == null && indexDefinition.getFields().size() >= orderedFields.size() && conditionHierarchy.size() == 1;
                        }
                        context.setVariable("$limit", limit);
                        cursor = operator.executeIndexQuery(context, index, keyParams, ascSortOrder);
                        if (!iSchemaClass.getName().equals(index.getDefinition().getClassName())) {
                            indexOnExactClass = false;
                        }
                    } catch (OIndexEngineException e) {
                        throw e;
                    } catch (Exception e) {
                        OLogManager.instance().error(this, "Error on using index %s in query '%s'. Probably you need to rebuild indexes. Now executing query using cluster scan", e, index.getName(), request != null && request.getText() != null ? request.getText() : "");
                        fullySortedByIndex = false;
                        cursors.clear();
                        return false;
                    }
                    if (cursor == null) {
                        continue;
                    }
                    if (index.getRebuildVersion() == indexRebuildVersion) {
                        cursors.add(OIndexChangesWrapper.wrap(index, cursor, indexRebuildVersion));
                        indexUseAttempts.add(new IndexUsageLog(index, keyParams, indexDefinition));
                        indexUsed = true;
                        break;
                    }
                }
                if (indexUsed) {
                    break;
                }
            }
            if (!indexUsed) {
                return tryOptimizeSort(iSchemaClass);
            }
        }
        if (cursors.size() == 0 || lastSearchResult == null) {
            return false;
        }
        if (cursors.size() == 1 && canOptimize(conditionHierarchy)) {
            filterOptimizer.optimize(compiledFilter, lastSearchResult);
        }
        uniqueResult = new ConcurrentHashMap<ORID, ORID>();
        if (cursors.size() == 1 && (compiledFilter == null || compiledFilter.getRootCondition() == null) && groupByFields == null && projections != null && projections.size() == 1) {
            // OPTIMIZATION: ONE INDEX USED WITH JUST ONE CONDITION: REMOVE THE FILTER
            final Entry<String, Object> entry = projections.entrySet().iterator().next();
            if (entry.getValue() instanceof OSQLFunctionRuntime) {
                final OSQLFunctionRuntime rf = (OSQLFunctionRuntime) entry.getValue();
                if (rf.function instanceof OSQLFunctionCount && rf.configuredParameters.length == 1 && "*".equals(rf.configuredParameters[0])) {
                    final boolean restrictedClasses = isUsingRestrictedClasses();
                    if (!restrictedClasses && indexOnExactClass) {
                        final OIndexCursor cursor = cursors.get(0);
                        long count = 0;
                        if (cursor instanceof OSizeable)
                            count = ((OSizeable) cursor).size();
                        else {
                            while (cursor.hasNext()) {
                                cursor.next();
                                count++;
                            }
                        }
                        final OProfiler profiler = Orient.instance().getProfiler();
                        if (profiler.isRecording()) {
                            profiler.updateCounter(profiler.getDatabaseMetric(database.getName(), "query.indexUsed"), "Used index in query", +1);
                        }
                        if (tempResult == null)
                            tempResult = new ArrayList<OIdentifiable>();
                        ((Collection<OIdentifiable>) tempResult).add(new ODocument().field(entry.getKey(), count));
                        return true;
                    }
                }
            }
        }
        for (OIndexCursor cursor : cursors) {
            if (!fetchValuesFromIndexCursor(cursor)) {
                break;
            }
        }
        uniqueResult.clear();
        uniqueResult = null;
        metricRecorder.recordOrderByOptimizationMetric(indexIsUsedInOrderBy, this.fullySortedByIndex);
        indexUseAttempts.clear();
        return true;
    } finally {
        for (IndexUsageLog wastedIndexUsage : indexUseAttempts) {
            revertProfiler(context, wastedIndexUsage.index, wastedIndexUsage.keyParams, wastedIndexUsage.indexDefinition);
        }
    }
}