Search in sources :

Example 1 with AsyncAPI

use of com.yahoo.elide.async.models.AsyncAPI in project elide by yahoo.

the class AsyncAPICancelRunnable method cancelAsyncAPI.

/**
 * This method cancels queries based on threshold.
 * @param type AsyncAPI Type Implementation.
 */
protected <T extends AsyncAPI> void cancelAsyncAPI(Class<T> type) {
    try {
        TransactionRegistry transactionRegistry = elide.getTransactionRegistry();
        Map<UUID, DataStoreTransaction> runningTransactionMap = transactionRegistry.getRunningTransactions();
        // Running transaction UUIDs
        Set<UUID> runningTransactionUUIDs = runningTransactionMap.keySet();
        // Construct filter expression
        PathElement statusPathElement = new PathElement(type, QueryStatus.class, "status");
        FilterExpression fltStatusExpression = new InPredicate(statusPathElement, QueryStatus.CANCELLED, QueryStatus.PROCESSING, QueryStatus.QUEUED);
        Iterable<T> asyncAPIIterable = asyncAPIDao.loadAsyncAPIByFilter(fltStatusExpression, type);
        // Active AsyncAPI UUIDs
        Set<UUID> asyncTransactionUUIDs = StreamSupport.stream(asyncAPIIterable.spliterator(), false).filter(query -> query.getStatus() == QueryStatus.CANCELLED || TimeUnit.SECONDS.convert(Math.abs(new Date(System.currentTimeMillis()).getTime() - query.getCreatedOn().getTime()), TimeUnit.MILLISECONDS) > maxRunTimeSeconds).map(query -> UUID.fromString(query.getRequestId())).collect(Collectors.toSet());
        // AsyncAPI UUIDs that have active transactions
        Set<UUID> queryUUIDsToCancel = Sets.intersection(runningTransactionUUIDs, asyncTransactionUUIDs);
        // AsyncAPI IDs that need to be cancelled
        Set<String> queryIDsToCancel = queryUUIDsToCancel.stream().map(uuid -> StreamSupport.stream(asyncAPIIterable.spliterator(), false).filter(query -> query.getRequestId().equals(uuid.toString())).map(T::getId).findFirst().orElseThrow(IllegalStateException::new)).collect(Collectors.toSet());
        // Cancel Transactions
        queryUUIDsToCancel.stream().forEach((uuid) -> {
            DataStoreTransaction runningTransaction = transactionRegistry.getRunningTransaction(uuid);
            if (runningTransaction != null) {
                JsonApiDocument jsonApiDoc = new JsonApiDocument();
                MultivaluedMap<String, String> queryParams = new MultivaluedHashMap<>();
                RequestScope scope = new RequestScope("", "query", NO_VERSION, jsonApiDoc, runningTransaction, null, queryParams, Collections.emptyMap(), uuid, elide.getElideSettings());
                runningTransaction.cancel(scope);
            }
        });
        // Change queryStatus for cancelled queries
        if (!queryIDsToCancel.isEmpty()) {
            PathElement idPathElement = new PathElement(type, String.class, "id");
            FilterExpression fltIdExpression = new InPredicate(idPathElement, queryIDsToCancel);
            asyncAPIDao.updateStatusAsyncAPIByFilter(fltIdExpression, QueryStatus.CANCEL_COMPLETE, type);
        }
    } catch (Exception e) {
        log.error("Exception in scheduled cancellation: {}", e.toString());
    }
}
Also used : TransactionRegistry(com.yahoo.elide.core.TransactionRegistry) JsonApiDocument(com.yahoo.elide.jsonapi.models.JsonApiDocument) Date(java.util.Date) AsyncAPIDAO(com.yahoo.elide.async.service.dao.AsyncAPIDAO) Map(java.util.Map) NO_VERSION(com.yahoo.elide.core.dictionary.EntityDictionary.NO_VERSION) StreamSupport(java.util.stream.StreamSupport) AsyncAPI(com.yahoo.elide.async.models.AsyncAPI) FilterExpression(com.yahoo.elide.core.filter.expression.FilterExpression) QueryStatus(com.yahoo.elide.async.models.QueryStatus) RequestScope(com.yahoo.elide.core.RequestScope) Elide(com.yahoo.elide.Elide) DataStoreTransaction(com.yahoo.elide.core.datastore.DataStoreTransaction) InPredicate(com.yahoo.elide.core.filter.predicates.InPredicate) AsyncQuery(com.yahoo.elide.async.models.AsyncQuery) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) Sets(com.google.common.collect.Sets) TimeUnit(java.util.concurrent.TimeUnit) MultivaluedMap(javax.ws.rs.core.MultivaluedMap) Slf4j(lombok.extern.slf4j.Slf4j) Data(lombok.Data) AllArgsConstructor(lombok.AllArgsConstructor) PathElement(com.yahoo.elide.core.Path.PathElement) Collections(java.util.Collections) JsonApiDocument(com.yahoo.elide.jsonapi.models.JsonApiDocument) TransactionRegistry(com.yahoo.elide.core.TransactionRegistry) InPredicate(com.yahoo.elide.core.filter.predicates.InPredicate) RequestScope(com.yahoo.elide.core.RequestScope) Date(java.util.Date) MultivaluedHashMap(javax.ws.rs.core.MultivaluedHashMap) PathElement(com.yahoo.elide.core.Path.PathElement) DataStoreTransaction(com.yahoo.elide.core.datastore.DataStoreTransaction) UUID(java.util.UUID) FilterExpression(com.yahoo.elide.core.filter.expression.FilterExpression)

Example 2 with AsyncAPI

use of com.yahoo.elide.async.models.AsyncAPI in project elide by yahoo.

the class ElideAsyncConfiguration method getTableExportHook.

// TODO Remove this method when ElideSettings has all the settings.
// Then the check can be done in TableExportHook.
// Trying to avoid adding too many individual properties to ElideSettings for now.
// https://github.com/yahoo/elide/issues/1803
private TableExportHook getTableExportHook(AsyncExecutorService asyncExecutorService, ElideConfigProperties settings, Map<ResultType, TableExportFormatter> supportedFormatters, ResultStorageEngine resultStorageEngine) {
    boolean exportEnabled = ElideAutoConfiguration.isExportEnabled(settings.getAsync());
    TableExportHook tableExportHook = null;
    if (exportEnabled) {
        tableExportHook = new TableExportHook(asyncExecutorService, settings.getAsync().getMaxAsyncAfterSeconds(), supportedFormatters, resultStorageEngine);
    } else {
        tableExportHook = new TableExportHook(asyncExecutorService, settings.getAsync().getMaxAsyncAfterSeconds(), supportedFormatters, resultStorageEngine) {

            @Override
            public void validateOptions(AsyncAPI export, RequestScope requestScope) {
                throw new InvalidOperationException("TableExport is not supported.");
            }
        };
    }
    return tableExportHook;
}
Also used : AsyncAPI(com.yahoo.elide.async.models.AsyncAPI) TableExportHook(com.yahoo.elide.async.hooks.TableExportHook) InvalidOperationException(com.yahoo.elide.core.exceptions.InvalidOperationException) RequestScope(com.yahoo.elide.core.security.RequestScope)

Example 3 with AsyncAPI

use of com.yahoo.elide.async.models.AsyncAPI in project elide by yahoo.

the class AsyncAPICancelRunnableTest method testTimeBasedCancellation.

@Test
public void testTimeBasedCancellation() {
    DataStoreTransaction dtx = elide.getDataStore().beginTransaction();
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf828d"), dtx);
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf827d"), dtx);
    AsyncQuery asyncQuery1 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf828d", System.currentTimeMillis(), QueryStatus.QUEUED);
    AsyncQuery asyncQuery2 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf827d", 1577883600000L, QueryStatus.QUEUED);
    AsyncQuery asyncQuery3 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf826d", 1577883600000L, QueryStatus.QUEUED);
    Collection<AsyncAPI> asyncCollection = new ArrayList<>();
    asyncCollection.add(asyncQuery1);
    asyncCollection.add(asyncQuery2);
    asyncCollection.add(asyncQuery3);
    when(cancelThread.getAsyncAPIDao().loadAsyncAPIByFilter(any(), any())).thenReturn(asyncCollection);
    cancelThread.cancelAsyncAPI(AsyncQuery.class);
    ArgumentCaptor<FilterExpression> filterCaptor = ArgumentCaptor.forClass(FilterExpression.class);
    ArgumentCaptor<QueryStatus> statusCaptor = ArgumentCaptor.forClass(QueryStatus.class);
    verify(asyncAPIDao, times(1)).updateStatusAsyncAPIByFilter(filterCaptor.capture(), statusCaptor.capture(), any());
    assertEquals("asyncQuery.id IN [[edc4a871-dff2-4054-804e-d80075cf827d]]", filterCaptor.getValue().toString());
    assertEquals("CANCEL_COMPLETE", statusCaptor.getValue().toString());
}
Also used : AsyncAPI(com.yahoo.elide.async.models.AsyncAPI) AsyncQuery(com.yahoo.elide.async.models.AsyncQuery) ArrayList(java.util.ArrayList) DataStoreTransaction(com.yahoo.elide.core.datastore.DataStoreTransaction) FilterExpression(com.yahoo.elide.core.filter.expression.FilterExpression) QueryStatus(com.yahoo.elide.async.models.QueryStatus) Test(org.junit.jupiter.api.Test)

Example 4 with AsyncAPI

use of com.yahoo.elide.async.models.AsyncAPI in project elide by yahoo.

the class AsyncAPICancelRunnableTest method testActiveTransactionCancellation.

@Test
public void testActiveTransactionCancellation() {
    DataStoreTransaction dtx = elide.getDataStore().beginTransaction();
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf828d"), dtx);
    AsyncQuery asyncQuery1 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf828d", 1577883600000L, QueryStatus.QUEUED);
    AsyncQuery asyncQuery2 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf827d", 1577883600000L, QueryStatus.QUEUED);
    Collection<AsyncAPI> asyncCollection = new ArrayList<>();
    asyncCollection.add(asyncQuery1);
    asyncCollection.add(asyncQuery2);
    when(cancelThread.getAsyncAPIDao().loadAsyncAPIByFilter(any(), any())).thenReturn(asyncCollection);
    cancelThread.cancelAsyncAPI(AsyncQuery.class);
    ArgumentCaptor<FilterExpression> filterCaptor = ArgumentCaptor.forClass(FilterExpression.class);
    ArgumentCaptor<QueryStatus> statusCaptor = ArgumentCaptor.forClass(QueryStatus.class);
    verify(asyncAPIDao, times(1)).loadAsyncAPIByFilter(any(), any());
    verify(asyncAPIDao, times(1)).updateStatusAsyncAPIByFilter(filterCaptor.capture(), statusCaptor.capture(), any());
    assertEquals("asyncQuery.id IN [[edc4a871-dff2-4054-804e-d80075cf828d]]", filterCaptor.getValue().toString());
    assertEquals("CANCEL_COMPLETE", statusCaptor.getValue().toString());
}
Also used : AsyncAPI(com.yahoo.elide.async.models.AsyncAPI) AsyncQuery(com.yahoo.elide.async.models.AsyncQuery) ArrayList(java.util.ArrayList) DataStoreTransaction(com.yahoo.elide.core.datastore.DataStoreTransaction) FilterExpression(com.yahoo.elide.core.filter.expression.FilterExpression) QueryStatus(com.yahoo.elide.async.models.QueryStatus) Test(org.junit.jupiter.api.Test)

Example 5 with AsyncAPI

use of com.yahoo.elide.async.models.AsyncAPI in project elide by yahoo.

the class AsyncAPICancelRunnableTest method testStatusBasedFilter.

@Test
public void testStatusBasedFilter() {
    DataStoreTransaction dtx = elide.getDataStore().beginTransaction();
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf828d"), dtx);
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf827d"), dtx);
    transactionRegistry.addRunningTransaction(UUID.fromString("edc4a871-dff2-4054-804e-d80075cf826d"), dtx);
    AsyncQuery asyncQuery1 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf828d", 1577883600000L, QueryStatus.CANCEL_COMPLETE);
    AsyncQuery asyncQuery2 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf827d", 1577883600000L, QueryStatus.CANCELLED);
    AsyncQuery asyncQuery3 = createAsyncQueryTestObject("edc4a871-dff2-4054-804e-d80075cf826d", 1577883600000L, QueryStatus.PROCESSING);
    Collection<AsyncAPI> asyncCollection = new ArrayList<>();
    asyncCollection.add(asyncQuery1);
    asyncCollection.add(asyncQuery2);
    asyncCollection.add(asyncQuery3);
    when(cancelThread.getAsyncAPIDao().loadAsyncAPIByFilter(any(), any())).thenReturn(asyncCollection);
    cancelThread.cancelAsyncAPI(AsyncQuery.class);
    ArgumentCaptor<FilterExpression> fltStatusCaptor = ArgumentCaptor.forClass(FilterExpression.class);
    verify(asyncAPIDao, times(1)).loadAsyncAPIByFilter(fltStatusCaptor.capture(), any());
    assertEquals("asyncQuery.status IN [CANCELLED, PROCESSING, QUEUED]", fltStatusCaptor.getValue().toString());
    verify(asyncAPIDao, times(1)).updateStatusAsyncAPIByFilter(any(), any(), any());
}
Also used : AsyncAPI(com.yahoo.elide.async.models.AsyncAPI) AsyncQuery(com.yahoo.elide.async.models.AsyncQuery) ArrayList(java.util.ArrayList) DataStoreTransaction(com.yahoo.elide.core.datastore.DataStoreTransaction) FilterExpression(com.yahoo.elide.core.filter.expression.FilterExpression) Test(org.junit.jupiter.api.Test)

Aggregations

AsyncAPI (com.yahoo.elide.async.models.AsyncAPI)5 AsyncQuery (com.yahoo.elide.async.models.AsyncQuery)4 DataStoreTransaction (com.yahoo.elide.core.datastore.DataStoreTransaction)4 FilterExpression (com.yahoo.elide.core.filter.expression.FilterExpression)4 QueryStatus (com.yahoo.elide.async.models.QueryStatus)3 ArrayList (java.util.ArrayList)3 Test (org.junit.jupiter.api.Test)3 Sets (com.google.common.collect.Sets)1 Elide (com.yahoo.elide.Elide)1 TableExportHook (com.yahoo.elide.async.hooks.TableExportHook)1 AsyncAPIDAO (com.yahoo.elide.async.service.dao.AsyncAPIDAO)1 PathElement (com.yahoo.elide.core.Path.PathElement)1 RequestScope (com.yahoo.elide.core.RequestScope)1 TransactionRegistry (com.yahoo.elide.core.TransactionRegistry)1 NO_VERSION (com.yahoo.elide.core.dictionary.EntityDictionary.NO_VERSION)1 InvalidOperationException (com.yahoo.elide.core.exceptions.InvalidOperationException)1 InPredicate (com.yahoo.elide.core.filter.predicates.InPredicate)1 RequestScope (com.yahoo.elide.core.security.RequestScope)1 JsonApiDocument (com.yahoo.elide.jsonapi.models.JsonApiDocument)1 Collections (java.util.Collections)1