Examples with IgniteDataStreamerTimeoutException org.apache.ignite.IgniteDataStreamerTimeoutException used on opensource projects

Example 1 with IgniteDataStreamerTimeoutException

use of org.apache.ignite.IgniteDataStreamerTimeoutException in project ignite by apache.

the class DataStreamerImpl method closeEx.

/**
 * @param cancel {@code True} to close with cancellation.
 * @param err Error.
 * @throws IgniteCheckedException If failed.
 */
private IgniteCheckedException closeEx(boolean cancel, IgniteCheckedException err) throws IgniteCheckedException {
    if (!closed.compareAndSet(false, true))
        return null;
    busyLock.writeLock();
    try {
        if (log.isDebugEnabled())
            log.debug("Closing data streamer [ldr=" + this + ", cancel=" + cancel + ']');
        try {
            // Assuming that no methods are called on this loader after this method is called.
            if (cancel) {
                cancelled = true;
                IgniteCheckedException cancellationErr = err;
                if (cancellationErr == null)
                    cancellationErr = new IgniteCheckedException("Data streamer has been cancelled: " + DataStreamerImpl.this);
                for (ThreadBuffer buf : threadBufMap.values()) {
                    GridFutureAdapter internalFut = (GridFutureAdapter) buf.getFuture().internalFuture();
                    internalFut.onDone(cancellationErr);
                }
                for (Buffer buf : bufMappings.values()) buf.cancelAll(cancellationErr);
            } else
                doFlush();
            ctx.event().removeLocalEventListener(discoLsnr);
            ctx.io().removeMessageListener(topic);
        } catch (IgniteCheckedException | IgniteDataStreamerTimeoutException e) {
            fut.onDone(e);
            throw e;
        }
        long failed = failCntr.longValue();
        if (failed > 0 && err == null)
            err = new IgniteCheckedException("Some of DataStreamer operations failed [failedCount=" + failed + "]");
        fut.onDone(err);
        return err;
    } finally {
        busyLock.writeUnlock();
    }
}

Example 2 with IgniteDataStreamerTimeoutException

use of org.apache.ignite.IgniteDataStreamerTimeoutException in project ignite by apache.

the class DataStreamerImpl method addDataInternal.

/**
 * @param entries Entries.
 * @param useThreadBuffer
 * @return Future.
 */
public IgniteFuture<?> addDataInternal(Collection<? extends DataStreamerEntry> entries, boolean useThreadBuffer) {
    checkSecurityPermissions(entries);
    IgniteCacheFutureImpl fut = null;
    GridFutureAdapter internalFut = null;
    Collection entriesList;
    lock(false);
    try {
        long threadId = Thread.currentThread().getId();
        if (useThreadBuffer) {
            ThreadBuffer threadBuf = threadBufMap.get(threadId);
            if (threadBuf == null) {
                fut = createDataLoadFuture();
                // Initial capacity should be more than batch by 12.5% in order to avoid resizing.
                threadBuf = new ThreadBuffer(fut, new ArrayList<>(bufLdrSzPerThread + (bufLdrSzPerThread >> 3)));
                threadBufMap.put(threadId, threadBuf);
            } else
                // Use existed thread-buffer future.
                fut = threadBuf.getFuture();
            entriesList = threadBuf.getEntries();
            entriesList.addAll(entries);
        } else {
            entriesList = entries;
            fut = createDataLoadFuture();
        }
        internalFut = (GridFutureAdapter) fut.internalFuture();
        if (!useThreadBuffer || entriesList.size() >= bufLdrSzPerThread) {
            loadData(entriesList, internalFut);
            if (useThreadBuffer)
                threadBufMap.remove(threadId);
        }
        return fut;
    } catch (Throwable e) {
        if (internalFut != null)
            internalFut.onDone(e);
        if (e instanceof Error || e instanceof IgniteDataStreamerTimeoutException)
            throw e;
        return fut;
    } finally {
        unlock(false);
    }
}

Example 3 with IgniteDataStreamerTimeoutException

use of org.apache.ignite.IgniteDataStreamerTimeoutException in project ignite by apache.

the class DataStreamerTimeoutTest method timeoutOnAddData.

/**
 */
private int timeoutOnAddData() throws Exception {
    boolean thrown = false;
    int processed = 0;
    try {
        Ignite ignite = startGrid(1);
        try (IgniteDataStreamer ldr = ignite.dataStreamer(CACHE_NAME)) {
            ldr.timeout(TIMEOUT);
            ldr.receiver(new TestDataReceiver());
            ldr.perThreadBufferSize(1);
            ldr.perNodeBufferSize(1);
            ldr.perNodeParallelOperations(1);
            ((DataStreamerImpl) ldr).maxRemapCount(0);
            try {
                for (int i = 0; i < ENTRY_AMOUNT; i++) {
                    ldr.addData(i, i);
                    processed++;
                }
            } catch (IllegalStateException ignored) {
            // No-op.
            }
        } catch (CacheException | IgniteDataStreamerTimeoutException ignored) {
            thrown = true;
        }
    } finally {
        stopAllGrids();
    }
    assertTrue(thrown);
    return processed;
}

Example 4 with IgniteDataStreamerTimeoutException

use of org.apache.ignite.IgniteDataStreamerTimeoutException in project ignite by apache.

the class DataStreamerImpl method doFlush.

/**
 * Performs flush.
 *
 * @throws IgniteCheckedException If failed.
 */
private void doFlush() throws IgniteCheckedException {
    lastFlushTime = U.currentTimeMillis();
    List<IgniteInternalFuture> activeFuts0 = null;
    int doneCnt = 0;
    flushAllThreadsBufs();
    for (IgniteInternalFuture<?> f : activeFuts) {
        if (!f.isDone()) {
            if (activeFuts0 == null)
                activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2));
            activeFuts0.add(f);
        } else {
            f.get();
            doneCnt++;
        }
    }
    if (activeFuts0 == null || activeFuts0.isEmpty())
        return;
    while (true) {
        if (disconnectErr != null)
            throw disconnectErr;
        Queue<IgniteInternalFuture<?>> q = null;
        for (Buffer buf : bufMappings.values()) {
            IgniteInternalFuture<?> flushFut = buf.flush();
            if (flushFut != null) {
                if (q == null)
                    q = new ArrayDeque<>(bufMappings.size() * 2);
                q.add(flushFut);
            }
        }
        if (q != null) {
            assert !q.isEmpty();
            boolean err = false;
            long startTimeMillis = U.currentTimeMillis();
            for (IgniteInternalFuture fut = q.poll(); fut != null; fut = q.poll()) {
                try {
                    if (timeout == DFLT_UNLIMIT_TIMEOUT)
                        fut.get();
                    else {
                        long timeRemain = timeout - U.currentTimeMillis() + startTimeMillis;
                        if (timeRemain <= 0)
                            throw new IgniteDataStreamerTimeoutException("Data streamer exceeded timeout on flush.");
                        fut.get(timeRemain);
                    }
                } catch (IgniteClientDisconnectedCheckedException e) {
                    if (log.isDebugEnabled())
                        log.debug("Failed to flush buffer: " + e);
                    throw CU.convertToCacheException(e);
                } catch (IgniteFutureTimeoutCheckedException e) {
                    if (log.isDebugEnabled())
                        log.debug("Failed to flush buffer: " + e);
                    throw new IgniteDataStreamerTimeoutException("Data streamer exceeded timeout on flush.", e);
                } catch (IgniteCheckedException e) {
                    if (log.isDebugEnabled())
                        log.debug("Failed to flush buffer: " + e);
                    err = true;
                    if (X.cause(e, IgniteClusterReadOnlyException.class) != null)
                        throw e;
                }
            }
            if (err)
                // Remaps needed - flush buffers.
                continue;
        }
        doneCnt = 0;
        for (int i = 0; i < activeFuts0.size(); i++) {
            IgniteInternalFuture f = activeFuts0.get(i);
            if (f == null)
                doneCnt++;
            else if (f.isDone()) {
                f.get();
                doneCnt++;
                activeFuts0.set(i, null);
            } else
                break;
        }
        if (doneCnt == activeFuts0.size())
            return;
    }
}

Example 5 with IgniteDataStreamerTimeoutException

use of org.apache.ignite.IgniteDataStreamerTimeoutException in project ignite by apache.

the class DataStreamerImpl method acquireRemapSemaphore.

/**
 */
private void acquireRemapSemaphore() throws IgniteInterruptedCheckedException {
    try {
        if (remapSem.availablePermits() != REMAP_SEMAPHORE_PERMISSIONS_COUNT) {
            if (timeout == DFLT_UNLIMIT_TIMEOUT) {
                // Wait until failed data being processed.
                remapSem.acquire(REMAP_SEMAPHORE_PERMISSIONS_COUNT);
                remapSem.release(REMAP_SEMAPHORE_PERMISSIONS_COUNT);
            } else {
                // Wait until failed data being processed.
                boolean res = remapSem.tryAcquire(REMAP_SEMAPHORE_PERMISSIONS_COUNT, timeout, TimeUnit.MILLISECONDS);
                if (res)
                    remapSem.release(REMAP_SEMAPHORE_PERMISSIONS_COUNT);
                else
                    throw new IgniteDataStreamerTimeoutException("Data streamer exceeded timeout " + "while was waiting for failed data resending finished.");
            }
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new IgniteInterruptedCheckedException(e);
    }
}