Examples with Table com.airbnb.airpal.presto.Table used on opensource projects

Example 1 with Table

use of com.airbnb.airpal.presto.Table in project airpal by airbnb.

the class Execution method doExecute.

private Job doExecute() throws ExecutionFailureException {
    final String userQuery = QUERY_SPLITTER.splitToList(getJob().getQuery()).get(0);
    final JobOutputBuilder outputBuilder;
    job.setQueryStats(createNoOpQueryStats());
    try {
        outputBuilder = outputBuilderFactory.forJob(job);
    } catch (IOException e) {
        throw new ExecutionFailureException(job, "Could not create output builder for job", e);
    } catch (InvalidQueryException e) {
        throw new ExecutionFailureException(job, e.getMessage(), e);
    }
    final Persistor persistor = persistorFactory.getPersistor(job, job.getOutput());
    final String query = job.getOutput().processQuery(userQuery);
    if (!persistor.canPersist(authorizer)) {
        throw new ExecutionFailureException(job, "Not authorized to create tables", null);
    }
    final List<List<Object>> outputPreview = new ArrayList<>(maxRowsPreviewOutput);
    final Set<Table> tables = new HashSet<>();
    try {
        tables.addAll(authorizer.tablesUsedByQuery(query));
    } catch (ParsingException e) {
        job.setError(new QueryError(e.getMessage(), null, -1, null, null, new ErrorLocation(e.getLineNumber(), e.getColumnNumber()), null));
        throw new ExecutionFailureException(job, "Invalid query, could not parse", e);
    }
    if (!authorizer.isAuthorizedRead(tables)) {
        job.setQueryStats(createNoOpQueryStats());
        throw new ExecutionFailureException(job, "Cannot access tables", null);
    }
    QueryClient queryClient = new QueryClient(queryRunner, timeout, query);
    try {
        queryClient.executeWith(new Function<StatementClient, Void>() {

            @Nullable
            @Override
            public Void apply(@Nullable StatementClient client) {
                if (client == null) {
                    return null;
                }
                QueryResults results = client.current();
                List<Column> resultColumns = null;
                JobState jobState = null;
                QueryError queryError = null;
                QueryStats queryStats = null;
                if (isCancelled) {
                    throw new ExecutionFailureException(job, "Query was cancelled", null);
                }
                if (results.getError() != null) {
                    queryError = results.getError();
                    jobState = JobState.FAILED;
                }
                if ((results.getInfoUri() != null) && (jobState != JobState.FAILED)) {
                    BasicQueryInfo queryInfo = queryInfoClient.from(results.getInfoUri());
                    if (queryInfo != null) {
                        queryStats = queryInfo.getQueryStats();
                    }
                }
                if (results.getStats() != null) {
                    jobState = JobState.fromStatementState(results.getStats().getState());
                }
                try {
                    if (results.getColumns() != null) {
                        resultColumns = results.getColumns();
                        outputBuilder.addColumns(resultColumns);
                    }
                    if (results.getData() != null) {
                        List<List<Object>> resultsData = ImmutableList.copyOf(results.getData());
                        for (List<Object> row : resultsData) {
                            outputBuilder.addRow(row);
                        }
                    }
                } catch (FileTooLargeException e) {
                    throw new ExecutionFailureException(job, "Output file exceeded maximum configured filesize", e);
                }
                rlUpdateJobInfo(tables, resultColumns, queryStats, jobState, queryError, outputPreview);
                return null;
            }
        });
    } catch (QueryTimeOutException e) {
        throw new ExecutionFailureException(job, format("Query exceeded maximum execution time of %s minutes", Duration.millis(e.getElapsedMs()).getStandardMinutes()), e);
    }
    QueryResults finalResults = queryClient.finalResults();
    if (finalResults != null && finalResults.getInfoUri() != null) {
        BasicQueryInfo queryInfo = queryInfoClient.from(finalResults.getInfoUri());
        if (queryInfo != null) {
            updateJobInfo(null, null, queryInfo.getQueryStats(), JobState.fromStatementState(finalResults.getStats().getState()), finalResults.getError(), outputPreview, true);
        }
    }
    if (job.getState() != JobState.FAILED) {
        URI location = persistor.persist(outputBuilder, job);
        if (location != null) {
            job.getOutput().setLocation(location);
        }
    } else {
        throw new ExecutionFailureException(job, null, null);
    }
    return getJob();
}

Example 2 with Table

use of com.airbnb.airpal.presto.Table in project airpal by airbnb.

the class ExecutionClient method jobFinished.

protected void jobFinished(Job job) {
    job.setQueryFinished(new DateTime());
    activeJobsStore.jobFinished(job);
    historyStore.addRun(job);
    for (Table t : job.getTablesUsed()) {
        usageStore.markUsage(t);
    }
    if (job.getOutput() instanceof HiveTablePersistentOutput && job.getOutput().getLocation() != null) {
        String[] parts = job.getOutput().getLocation().toString().split("\\.");
        if (parts.length == 2) {
            Map<String, List<String>> cache = schemaCache.getSchemaMap("hive");
            List<String> tables = cache.get(parts[0]);
            tables.add(parts[1]);
        }
    }
    eventBus.post(new JobFinishedEvent(job));
    executionMap.remove(job.getUuid());
}

Example 3 with Table

use of com.airbnb.airpal.presto.Table in project airpal by airbnb.

the class QueriesResource method getQueries.

@GET
public Response getQueries(@Auth AirpalUser user, @QueryParam("results") int numResults, @QueryParam("table") List<PartitionedTable> tables) {
    Iterable<Job> recentlyRun;
    int results = Optional.of(numResults).or(200);
    if (tables.size() < 1) {
        recentlyRun = jobHistoryStore.getRecentlyRun(results);
    } else {
        recentlyRun = jobHistoryStore.getRecentlyRun(results, Iterables.transform(tables, new PartitionedTable.PartitionedTableToTable()));
    }
    ImmutableList.Builder<Job> filtered = ImmutableList.builder();
    for (Job job : recentlyRun) {
        if (job.getTablesUsed().isEmpty() && (job.getState() == JobState.FAILED)) {
            filtered.add(job);
            continue;
        }
        for (Table table : job.getTablesUsed()) {
            if (AuthorizationUtil.isAuthorizedRead(user, table)) {
                filtered.add(new Job(job.getUser(), job.getQuery(), job.getUuid(), job.getOutput(), job.getQueryStats(), job.getState(), Collections.<Column>emptyList(), Collections.<Table>emptySet(), job.getQueryStartedDateTime(), job.getError(), job.getQueryFinishedDateTime()));
            }
        }
    }
    List<Job> sortedResult = Ordering.natural().nullsLast().onResultOf(JOB_ORDERING).reverse().immutableSortedCopy(filtered.build());
    return Response.ok(sortedResult).build();
}

Example 4 with Table

use of com.airbnb.airpal.presto.Table in project airpal by airbnb.

the class TablesResource method getTableUpdates.

@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getTableUpdates(@Auth AirpalUser user, @QueryParam("catalog") Optional<String> catalogOptional) {
    final String catalog = catalogOptional.or(defaultCatalog);
    final Map<String, List<String>> schemaMap = schemaCache.getSchemaMap(catalog);
    final ImmutableList.Builder<Table> builder = ImmutableList.builder();
    for (Map.Entry<String, List<String>> entry : schemaMap.entrySet()) {
        String schema = entry.getKey();
        for (String table : entry.getValue()) {
            if (isAuthorizedRead(user, catalog, schema, table)) {
                builder.add(new Table(catalog, schema, table));
            }
        }
    }
    final List<Table> tables = builder.build();
    final Map<Table, Long> allUsages = usageStore.getUsages(tables);
    final Map<PartitionedTable, DateTime> updateMap = Collections.emptyMap();
    return Response.ok(createTablesWithMetaData(tables, allUsages, updateMap)).build();
}

Example 5 with Table

use of com.airbnb.airpal.presto.Table in project airpal by airbnb.

the class UsersResource method getUserQueries.

@GET
@Path("queries")
public Response getUserQueries(@Auth AirpalUser user, @PathParam("id") String userId, @QueryParam("results") int numResults, @QueryParam("table") List<PartitionedTable> tables) {
    Iterable<Job> recentlyRun;
    int results = Optional.of(numResults).or(0);
    if (results <= 0) {
        results = 100;
    }
    if (tables.size() < 1) {
        recentlyRun = jobHistoryStore.getRecentlyRunForUser(userId, results);
    } else {
        recentlyRun = jobHistoryStore.getRecentlyRunForUser(userId, results, Iterables.transform(tables, new PartitionedTableToTable()));
    }
    ImmutableList.Builder<Job> filtered = ImmutableList.builder();
    for (Job job : recentlyRun) {
        if (job.getTablesUsed().isEmpty() && (job.getState() == JobState.FAILED)) {
            filtered.add(job);
            continue;
        }
        for (Table table : job.getTablesUsed()) {
            if (AuthorizationUtil.isAuthorizedRead(user, table)) {
                filtered.add(new Job(job.getUser(), job.getQuery(), job.getUuid(), job.getOutput(), job.getQueryStats(), job.getState(), Collections.<Column>emptyList(), Collections.<Table>emptySet(), job.getQueryStartedDateTime(), job.getError(), job.getQueryFinishedDateTime()));
            }
        }
    }
    List<Job> sortedResult = Ordering.natural().nullsLast().onResultOf(JOB_ORDERING).reverse().immutableSortedCopy(filtered.build());
    return Response.ok(sortedResult).build();
}