Examples with QueryData annis.ql.parser.QueryData used on opensource projects

Example 6 with QueryData

use of annis.ql.parser.QueryData in project ANNIS by korpling.

the class TestAbstractSqlGenerator method shouldAppendLimitAndOffsetClause.

/**
 * Append LIMIT/OFFSET clause.
 */
@Test
public void shouldAppendLimitAndOffsetClause() {
    // given
    String selectClause = uniqueString();
    String fromClause = uniqueString();
    setupSelectAndFromClause(selectClause, fromClause);
    String limitOffset = uniqueString();
    @SuppressWarnings("unchecked") LimitOffsetClauseSqlGenerator<QueryData> limitOffsetClauseSqlGenerator = mock(LimitOffsetClauseSqlGenerator.class);
    generator.setLimitOffsetClauseSqlGenerator(limitOffsetClauseSqlGenerator);
    given(limitOffsetClauseSqlGenerator.limitOffsetClause(eq(queryData), eq(alternative), anyString())).willReturn(limitOffset);
    // when
    String sql = generator.toSql(queryData);
    // then
    String expected = createMinimalSqlStatement(selectClause, fromClause);
    expected += limitOffset + "\n";
    assertThat(sql, is(expected));
}

Example 7 with QueryData

use of annis.ql.parser.QueryData in project ANNIS by korpling.

the class AnnisRunner method doSubgraph.

public void doSubgraph(String saltIds) {
    QueryData queryData = analyzeQuery(saltIds, "subgraph");
    out.println("NOTICE: left = " + left + "; right = " + right + "; limit = " + limit + "; offset = " + offset + "; filter = " + filter.name());
    SaltProject result = queryDao.graph(queryData);
    // write result to File
    URI path = URI.createFileURI("/tmp/annissalt");
    SaltUtil.save_DOT(result, path);
    System.out.println("graph as dot written to /tmp/annissalt");
}

Example 8 with QueryData

use of annis.ql.parser.QueryData in project ANNIS by korpling.

the class AnnisRunner method doSql.

// FIXME: missing tests
public void doSql(String funcCall) {
    String doSqlFunctionName = "sql_" + funcCall.split("\\s", 2)[0];
    SqlGenerator<QueryData> gen = getGeneratorForQueryFunction(funcCall);
    String annisQuery = getAnnisQueryFromFunctionCall(funcCall);
    QueryData queryData = analyzeQuery(annisQuery, doSqlFunctionName);
    out.println("NOTICE: left = " + left + "; right = " + right + "; limit = " + limit + "; offset = " + offset);
    out.println(gen.toSql(queryData));
}

Example 9 with QueryData

use of annis.ql.parser.QueryData in project ANNIS by korpling.

the class AnnisRunner method doBenchmarkFile.

// /// Commands
public void doBenchmarkFile(String filename) {
    try {
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "UTF-8"))) {
            Map<String, Integer> queryRun = new HashMap<>();
            Map<String, Integer> queryId = new HashMap<>();
            int maxId = 0;
            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                // get the id of this query
                if (!queryId.containsKey(line)) {
                    ++maxId;
                    queryId.put(line, maxId);
                }
                int id = queryId.get(line);
                // get the repetition of this query
                if (!queryRun.containsKey(line)) {
                    queryRun.put(line, 0);
                }
                int run = queryRun.get(line) + 1;
                queryRun.put(line, run);
                String[] split = line.split(" ", 2);
                String queryFunction = split[0];
                String annisQuery = split[1];
                boolean error = false;
                QueryData queryData = null;
                try {
                    queryData = analyzeQuery(annisQuery, queryFunction);
                } catch (RuntimeException e) {
                    error = true;
                }
                if ("count".equals(queryFunction)) {
                    long start = new Date().getTime();
                    if (!error) {
                        try {
                            int result = queryDao.count(queryData);
                            long end = new Date().getTime();
                            long runtime = end - start;
                            Object[] output = { queryFunction, annisQuery, result, runtime, id, run };
                            System.out.println(StringUtils.join(output, "\t"));
                        } catch (RuntimeException e) {
                            error = true;
                        }
                    }
                    if (error) {
                        String result = "ERROR";
                        long end = new Date().getTime();
                        long runtime = end - start;
                        Object[] output = { queryFunction, annisQuery, result, runtime, id, run };
                        System.out.println(StringUtils.join(output, "\t"));
                    }
                }
            }
        }
    } catch (IOException e) {
        error(e);
    }
}

Example 10 with QueryData

use of annis.ql.parser.QueryData in project ANNIS by korpling.

the class AnnisRunner method doBenchmark.

public void doBenchmark(String benchmarkCount) {
    int count = Integer.parseInt(benchmarkCount);
    out.println("---> executing " + benchmarks.size() + " queries " + count + " times");
    AnnisRunner.OS currentOS = AnnisRunner.OS.other;
    try {
        currentOS = AnnisRunner.OS.valueOf(System.getProperty("os.name").toLowerCase());
    } catch (IllegalArgumentException ex) {
    }
    List<AnnisRunner.Benchmark> session = new ArrayList<>();
    // create sql + plan for each query and create count copies for each benchmark
    for (Benchmark benchmark : benchmarks) {
        if (clearCaches) {
            resetCaches(currentOS);
        }
        SqlGeneratorAndExtractor<QueryData, ?> generator = getGeneratorForQueryFunction(benchmark.functionCall);
        benchmark.sql = getGeneratorForQueryFunction(benchmark.functionCall).toSql(benchmark.queryData);
        out.println("---> SQL query for: " + benchmark.functionCall);
        out.println(benchmark.sql);
        try {
            benchmark.plan = queryDao.explain(generator, benchmark.queryData, false);
            out.println("---> query plan for: " + benchmark.functionCall);
            out.println(benchmark.plan);
        } catch (RuntimeException ex) {
            // nested DataAccessException would be better
            out.println("---> query plan failed for " + benchmark.functionCall);
        }
        benchmark.bestTimeInMilliseconds = Long.MAX_VALUE;
        benchmark.worstTimeInMilliseconds = Long.MIN_VALUE;
        out.println("---> running query sequentially " + SEQUENTIAL_RUNS + " times");
        String options = benchmarkOptions(benchmark.queryData);
        for (int i = 0; i < SEQUENTIAL_RUNS; ++i) {
            if (i > 0) {
                out.print(", ");
            }
            boolean error = false;
            long start = new Date().getTime();
            try {
                queryDao.executeQueryFunction(benchmark.queryData, generator);
            } catch (RuntimeException ex) {
                error = true;
            }
            long end = new Date().getTime();
            long runtime = end - start;
            if (benchMode == BenchmarkMode.sequential_random) {
                // record the runtime and other benchmark values when the sequental run is counted
                benchmark.sumTimeInMilliseconds += runtime;
                benchmark.values.add(runtime);
                benchmark.bestTimeInMilliseconds = Math.min(benchmark.bestTimeInMilliseconds, runtime);
                benchmark.worstTimeInMilliseconds = Math.max(benchmark.worstTimeInMilliseconds, runtime);
                ++benchmark.runs;
                if (error) {
                    ++benchmark.errors;
                }
            }
            out.print(runtime + " ms");
        }
        out.println();
        out.println(benchmark.bestTimeInMilliseconds + " ms best time for '" + benchmark.functionCall + ("".equals(options) ? "'" : "' with " + options));
        session.addAll(Collections.nCopies(count, benchmark));
    }
    // the others
    if (clearCaches) {
        resetCaches(currentOS);
    }
    // shuffle the benchmark queries
    Collections.shuffle(session);
    out.println();
    out.println("---> running queries in random order");
    // execute the random query order, record test times
    for (AnnisRunner.Benchmark benchmark : session) {
        if (benchmark.errors >= 3) {
            continue;
        }
        boolean error = false;
        SqlGeneratorAndExtractor<QueryData, ?> generator = getGeneratorForQueryFunction(benchmark.functionCall);
        long start = new Date().getTime();
        try {
            queryDao.executeQueryFunction(benchmark.queryData, generator);
        } catch (RuntimeException e) {
            error = true;
        }
        long end = new Date().getTime();
        long runtime = end - start;
        benchmark.sumTimeInMilliseconds += runtime;
        benchmark.values.add(runtime);
        benchmark.bestTimeInMilliseconds = Math.min(benchmark.bestTimeInMilliseconds, runtime);
        benchmark.worstTimeInMilliseconds = Math.max(benchmark.worstTimeInMilliseconds, runtime);
        ++benchmark.runs;
        if (error) {
            ++benchmark.errors;
        }
        String options = benchmarkOptions(benchmark.queryData);
        out.println(runtime + " ms for '" + benchmark.functionCall + ("".equals(options) ? "'" : "' with " + options) + (error ? " ERROR" : ""));
    }
    // compute average runtime for each query
    out.println();
    out.println("---> benchmark complete");
    for (AnnisRunner.Benchmark benchmark : benchmarks) {
        String options = benchmarkOptions(benchmark.queryData);
        out.println(benchmark.getMedian() + " ms (median for " + benchmark.runs + " runs" + (benchmark.errors > 0 ? ", " + benchmark.errors + " errors)" : ")") + " for '" + benchmark.functionCall + ("".equals(options) ? "'" : "' with " + options));
    }
    // show best runtime for each query
    out.println();
    out.println("---> worst times");
    for (AnnisRunner.Benchmark benchmark : benchmarks) {
        String options = benchmarkOptions(benchmark.queryData);
        out.println(benchmark.worstTimeInMilliseconds + " ms " + (benchmark.errors > 0 ? "(" + benchmark.errors + " errors)" : "") + " for '" + benchmark.functionCall + ("".equals(options) ? "'" : "' with " + options));
    }
    // show best runtime for each query
    out.println();
    out.println("---> best times");
    for (AnnisRunner.Benchmark benchmark : benchmarks) {
        String options = benchmarkOptions(benchmark.queryData);
        out.println(benchmark.bestTimeInMilliseconds + " ms " + (benchmark.errors > 0 ? "(" + benchmark.errors + " errors)" : "") + " for '" + benchmark.functionCall + ("".equals(options) ? "'" : "' with " + options));
    }
    out.println();
    // CSV output
    try (CSVWriter csv = new CSVWriter(new FileWriterWithEncoding(new File("annis_benchmark_result.csv"), "UTF-8"))) {
        String[] header = new String[] { "corpora", "query", "median", "diff-best", "diff-worst", "mean" };
        csv.writeNext(header);
        for (AnnisRunner.Benchmark benchmark : benchmarks) {
            double mean = (double) benchmark.sumTimeInMilliseconds / (double) benchmark.runs;
            long median = benchmark.getMedian();
            String[] line = new String[6];
            line[0] = StringUtils.join(benchmark.queryData.getCorpusList(), ",");
            line[1] = benchmark.functionCall;
            line[2] = "" + median;
            line[3] = "" + Math.abs(benchmark.bestTimeInMilliseconds - median);
            line[4] = "" + Math.abs(median - benchmark.worstTimeInMilliseconds);
            line[5] = "" + mean;
            csv.writeNext(line);
        }
    } catch (IOException ex) {
        log.error(null, ex);
    }
    // property output format for Jenkins Plot plugin
    try {
        File outputDir = new File("annis_benchmark_results");
        if (outputDir.isDirectory() || outputDir.mkdirs()) {
            int i = 1;
            for (AnnisRunner.Benchmark b : benchmarks) {
                Properties props = new Properties();
                props.put("YVALUE", "" + b.getMedian());
                try (FileWriterWithEncoding writer = new FileWriterWithEncoding(new File(outputDir, i + ".properties"), "UTF-8")) {
                    props.store(writer, "");
                }
                i++;
                // also write out a "time" and "count" file which can be used by the ANNIS4 prototype
                if (b.name != null) {
                    double mean = (double) b.sumTimeInMilliseconds / (double) b.runs;
                    Files.write("" + mean, new File(outputDir, b.name + ".time"), StandardCharsets.UTF_8);
                    if (b.count != null) {
                        Files.write("" + b.count, new File(outputDir, b.name + ".count"), StandardCharsets.UTF_8);
                    }
                }
            }
        }
    } catch (IOException ex) {
        log.error(null, ex);
    }
}