Examples with Stats org.apache.asterix.translator.IStatementExecutor.Stats used on opensource projects

Example 1 with Stats

use of org.apache.asterix.translator.IStatementExecutor.Stats in project asterixdb by apache.

the class APIFramework method compileQuery.

public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider, Query rwQ, int varCounter, String outputDatasetName, SessionOutput output, ICompiledDmlStatement statement) throws AlgebricksException, RemoteException, ACIDException {
    SessionConfig conf = output.config();
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
        output.out().println();
        printPlanPrefix(output, "Rewritten expression tree");
        if (rwQ != null) {
            rwQ.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
        }
        printPlanPostfix(output);
    }
    org.apache.asterix.common.transactions.JobId asterixJobId = JobIdFactory.generateJobId();
    metadataProvider.setJobId(asterixJobId);
    ILangExpressionToPlanTranslator t = translatorFactory.createExpressionToPlanTranslator(metadataProvider, varCounter);
    ILogicalPlan plan;
    // statement = null when it's a query
    if (statement == null || statement.getKind() != Statement.Kind.LOAD) {
        plan = t.translate(rwQ, outputDatasetName, statement);
    } else {
        plan = t.translateLoad(statement);
    }
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
        output.out().println();
        printPlanPrefix(output, "Logical plan");
        if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
        }
        printPlanPostfix(output);
    }
    CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties();
    int frameSize = compilerProperties.getFrameSize();
    Map<String, String> querySpecificConfig = metadataProvider.getConfig();
    // Validates the user-overridden query parameters.
    validateConfig(querySpecificConfig);
    int sortFrameLimit = getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), compilerProperties.getSortMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_SORT);
    int groupFrameLimit = getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY);
    int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY), compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesForJoin(joinFrameLimit);
    HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(OptimizationContextFactory.INSTANCE);
    builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
    builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext()));
    builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext()));
    IDataFormat format = metadataProvider.getFormat();
    ICompilerFactory compilerFactory = builder.create();
    builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());
    builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory());
    builder.setPartialAggregationTypeComputer(new PartialAggregationTypeComputer());
    builder.setExpressionTypeComputer(ExpressionTypeComputer.INSTANCE);
    builder.setMissableTypeComputer(MissableTypeComputer.INSTANCE);
    builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE);
    int parallelism = getParallelism(querySpecificConfig.get(CompilerProperties.COMPILER_PARALLELISM_KEY), compilerProperties.getParallelism());
    AlgebricksAbsolutePartitionConstraint computationLocations = chooseLocations(clusterInfoCollector, parallelism, metadataProvider.getClusterLocations());
    builder.setClusterLocations(computationLocations);
    ICompiler compiler = compilerFactory.createCompiler(plan, metadataProvider, t.getVarCounter());
    if (conf.isOptimize()) {
        compiler.optimize();
        if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)) {
            if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)) {
                // For Optimizer tests.
                AlgebricksAppendable buffer = new AlgebricksAppendable(output.out());
                PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
            } else {
                printPlanPrefix(output, "Optimized logical plan");
                if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
                    LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
                    PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
                }
                printPlanPostfix(output);
            }
        }
    }
    if (rwQ != null && rwQ.isExplain()) {
        try {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
            ResultUtil.printResults(metadataProvider.getApplicationContext(), pvisitor.get().toString(), output, new Stats(), null);
            return null;
        } catch (IOException e) {
            throw new AlgebricksException(e);
        }
    }
    if (!conf.isGenerateJobSpec()) {
        return null;
    }
    builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory());
    builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
    builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
    builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE));
    builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
    builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
    builder.setMissingWriterFactory(format.getMissingWriterFactory());
    builder.setPredicateEvaluatorFactoryProvider(format.getPredicateEvaluatorFactoryProvider());
    final SessionConfig.OutputFormat outputFormat = conf.fmt();
    switch(outputFormat) {
        case LOSSLESS_JSON:
            builder.setPrinterProvider(format.getLosslessJSONPrinterFactoryProvider());
            break;
        case CSV:
            builder.setPrinterProvider(format.getCSVPrinterFactoryProvider());
            break;
        case ADM:
            builder.setPrinterProvider(format.getADMPrinterFactoryProvider());
            break;
        case CLEAN_JSON:
            builder.setPrinterProvider(format.getCleanJSONPrinterFactoryProvider());
            break;
        default:
            throw new AlgebricksException("Unexpected OutputFormat: " + outputFormat);
    }
    builder.setSerializerDeserializerProvider(format.getSerdeProvider());
    builder.setTypeTraitProvider(format.getTypeTraitProvider());
    builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
    JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId, metadataProvider.isWriteTransaction());
    JobSpecification spec = compiler.createJob(metadataProvider.getApplicationContext(), jobEventListenerFactory);
    // When the top-level statement is a query, the statement parameter is null.
    if (statement == null) {
        // Sets a required capacity, only for read-only queries.
        // DDLs and DMLs are considered not that frequent.
        spec.setRequiredClusterCapacity(ResourceUtils.getRequiredCompacity(plan, computationLocations, sortFrameLimit, groupFrameLimit, joinFrameLimit, frameSize));
    }
    if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
        printPlanPrefix(output, "Hyracks job");
        if (rwQ != null) {
            try {
                output.out().println(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(spec.toJSON()));
            } catch (IOException e) {
                throw new AlgebricksException(e);
            }
            output.out().println(spec.getUserConstraints());
        }
        printPlanPostfix(output);
    }
    return spec;
}

Example 2 with Stats

use of org.apache.asterix.translator.IStatementExecutor.Stats in project asterixdb by apache.

the class QueryServiceServlet method handleRequest.

private void handleRequest(RequestParameters param, IServletResponse response) throws IOException {
    LOGGER.info(param.toString());
    long elapsedStart = System.nanoTime();
    final StringWriter stringWriter = new StringWriter();
    final PrintWriter resultWriter = new PrintWriter(stringWriter);
    ResultDelivery delivery = parseResultDelivery(param.mode);
    String handleUrl = getHandleUrl(param.host, param.path, delivery);
    SessionOutput sessionOutput = createSessionOutput(param, handleUrl, resultWriter);
    SessionConfig sessionConfig = sessionOutput.config();
    HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
    HttpResponseStatus status = HttpResponseStatus.OK;
    Stats stats = new Stats();
    long[] execStartEnd = new long[] { -1, -1 };
    resultWriter.print("{\n");
    printRequestId(resultWriter);
    printClientContextID(resultWriter, param);
    printSignature(resultWriter);
    printType(resultWriter, sessionConfig);
    try {
        if (param.statement == null || param.statement.isEmpty()) {
            throw new AsterixException("Empty request, no statement provided");
        }
        String statementsText = param.statement + ";";
        executeStatement(statementsText, sessionOutput, delivery, stats, param, handleUrl, execStartEnd);
        if (ResultDelivery.IMMEDIATE == delivery || ResultDelivery.DEFERRED == delivery) {
            ResultUtil.printStatus(sessionOutput, ResultStatus.SUCCESS);
        }
    } catch (AlgebricksException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.INFO, pe.getMessage(), pe);
        ResultUtil.printError(resultWriter, pe);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.BAD_REQUEST;
    } catch (HyracksException pe) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.WARNING, pe.getMessage(), pe);
        ResultUtil.printError(resultWriter, pe);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
    } catch (Exception e) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, "Unexpected exception", e);
        ResultUtil.printError(resultWriter, e);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
    } finally {
        if (execStartEnd[0] == -1) {
            execStartEnd[1] = -1;
        } else if (execStartEnd[1] == -1) {
            execStartEnd[1] = System.nanoTime();
        }
    }
    printMetrics(resultWriter, System.nanoTime() - elapsedStart, execStartEnd[1] - execStartEnd[0], stats.getCount(), stats.getSize());
    resultWriter.print("}\n");
    resultWriter.flush();
    String result = stringWriter.toString();
    GlobalConfig.ASTERIX_LOGGER.log(Level.FINE, result);
    response.setStatus(status);
    response.writer().print(result);
    if (response.writer().checkError()) {
        LOGGER.warning("Error flushing output writer");
    }
}

Example 3 with Stats

use of org.apache.asterix.translator.IStatementExecutor.Stats in project asterixdb by apache.

the class QueryResultApiServlet method get.

@Override
protected void get(IServletRequest request, IServletResponse response) throws Exception {
    // TODO this seems wrong ...
    HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, HttpUtil.Encoding.UTF8);
    PrintWriter out = response.writer();
    final String strHandle = localPath(request);
    final ResultHandle handle = ResultHandle.parse(strHandle);
    if (handle == null) {
        response.setStatus(HttpResponseStatus.BAD_REQUEST);
        return;
    }
    IHyracksDataset hds = getHyracksDataset();
    ResultReader resultReader = new ResultReader(hds, handle.getJobId(), handle.getResultSetId());
    try {
        DatasetJobRecord.Status status = resultReader.getStatus();
        final HttpResponseStatus httpStatus;
        if (status == null) {
            httpStatus = HttpResponseStatus.NOT_FOUND;
        } else {
            switch(status.getState()) {
                case SUCCESS:
                    httpStatus = HttpResponseStatus.OK;
                    break;
                case RUNNING:
                case IDLE:
                case FAILED:
                    httpStatus = HttpResponseStatus.NOT_FOUND;
                    break;
                default:
                    httpStatus = HttpResponseStatus.INTERNAL_SERVER_ERROR;
                    break;
            }
        }
        response.setStatus(httpStatus);
        if (httpStatus != HttpResponseStatus.OK) {
            return;
        }
        // QQQ The output format is determined by the initial
        // query and cannot be modified here, so calling back to
        // initResponse() is really an error. We need to find a
        // way to send the same OutputFormat value here as was
        // originally determined there. Need to save this value on
        // some object that we can obtain here.
        SessionOutput sessionOutput = RestApiServlet.initResponse(request, response);
        ResultUtil.printResults(appCtx, resultReader, sessionOutput, new Stats(), null);
    } catch (HyracksDataException e) {
        final int errorCode = e.getErrorCode();
        if (ErrorCode.NO_RESULTSET == errorCode) {
            LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
            response.setStatus(HttpResponseStatus.NOT_FOUND);
            return;
        }
        response.setStatus(HttpResponseStatus.BAD_REQUEST);
        out.println(e.getMessage());
        LOGGER.log(Level.WARNING, "Error retrieving result for \"" + strHandle + "\"", e);
    } catch (Exception e) {
        response.setStatus(HttpResponseStatus.BAD_REQUEST);
        LOGGER.log(Level.WARNING, "Error retrieving result for \"" + strHandle + "\"", e);
    }
    if (out.checkError()) {
        LOGGER.warning("Error flushing output writer for \"" + strHandle + "\"");
    }
}