Examples with SyncInfo alluxio.SyncInfo used on opensource projects

Example 1 with SyncInfo

use of alluxio.SyncInfo in project alluxio by Alluxio.

the class ActiveSyncer method heartbeat.

@Override
public void heartbeat() {
    LOG.debug("start sync heartbeat for {} with mount id {}", mMountUri, mMountId);
    // Remove any previously completed sync tasks
    mSyncTasks.removeIf(Future::isDone);
    List<AlluxioURI> filterList = mSyncManager.getFilterList(mMountId);
    if (filterList == null || filterList.isEmpty()) {
        return;
    }
    try {
        UfsManager.UfsClient ufsclient = Objects.requireNonNull(mMountTable.getUfsClient(mMountId));
        try (CloseableResource<UnderFileSystem> ufsResource = ufsclient.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            if (!ufs.supportsActiveSync()) {
                return;
            }
            SyncInfo syncInfo = ufs.getActiveSyncInfo();
            // This returns a list of ufsUris that we need to sync.
            Set<AlluxioURI> ufsSyncPoints = syncInfo.getSyncPoints();
            // Parallelize across sync points
            List<CompletableFuture<Long>> tasksPerSync = new ArrayList<>();
            for (AlluxioURI ufsUri : ufsSyncPoints) {
                tasksPerSync.add(CompletableFuture.supplyAsync(() -> {
                    processSyncPoint(ufsUri, syncInfo);
                    return syncInfo.getTxId();
                }, mSyncManager.getExecutor()));
            }
            // Journal the latest processed txId
            CompletableFuture<Void> syncTask = CompletableFuture.allOf(tasksPerSync.toArray(new CompletableFuture<?>[0])).thenRunAsync(() -> mFileSystemMaster.recordActiveSyncTxid(syncInfo.getTxId(), mMountId), mSyncManager.getExecutor());
            int attempts = 0;
            while (!mSyncTasks.offer(syncTask)) {
                if (Thread.currentThread().isInterrupted()) {
                    break;
                }
                // sync tasks in the last 32 / (heartbeats/minute) minutes have not completed.
                for (CompletableFuture<?> f : mSyncTasks) {
                    try {
                        long waitTime = ServerConfiguration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL) / mSyncTasks.size();
                        f.get(waitTime, TimeUnit.MILLISECONDS);
                        mSyncTasks.remove(f);
                        break;
                    } catch (TimeoutException e) {
                        LOG.trace("sync task did not complete during heartbeat. Attempt: {}", attempts);
                    } catch (InterruptedException | ExecutionException e) {
                        LogUtils.warnWithException(LOG, "Failed while waiting on task to add new task to " + "head of queue", e);
                        if (Thread.currentThread().isInterrupted()) {
                            Thread.currentThread().interrupt();
                        }
                    }
                    attempts++;
                }
            }
        }
    } catch (IOException e) {
        LOG.warn("IOException " + Throwables.getStackTraceAsString(e));
    }
}

Example 2 with SyncInfo

use of alluxio.SyncInfo in project alluxio by Alluxio.

the class SupportedHdfsActiveSyncProvider method getActivitySyncInfo.

/**
 * Get the activity sync info.
 *
 * @return SyncInfo object which encapsulates the necessary information about changes
 */
public SyncInfo getActivitySyncInfo() {
    if (mPollingThread == null) {
        return SyncInfo.emptyInfo();
    }
    Map<AlluxioURI, Set<AlluxioURI>> syncPointFiles = new HashMap<>();
    long txId = 0;
    try (LockResource r = new LockResource(mWriteLock)) {
        initNextWindow();
        if (mEventMissed) {
            // force sync every syncpoint
            for (AlluxioURI uri : mUfsUriList) {
                syncPointFiles.put(uri, null);
                syncSyncPoint(uri.toString());
            }
            mEventMissed = false;
            LOG.debug("Missed event, syncing all sync points\n{}", Arrays.toString(syncPointFiles.keySet().toArray()));
            SyncInfo syncInfo = new SyncInfo(syncPointFiles, true, getLastTxId());
            return syncInfo;
        }
        for (String syncPoint : mActivity.keySet()) {
            AlluxioURI syncPointURI = new AlluxioURI(syncPoint);
            // if the activity level is below the threshold or the sync point is too old, we sync
            if (mActivity.get(syncPoint) < mActiveUfsSyncMaxActivity || mAge.get(syncPoint) > mActiveUfsSyncMaxAge) {
                if (!syncPointFiles.containsKey(syncPointURI)) {
                    syncPointFiles.put(syncPointURI, mChangedFiles.get(syncPoint));
                }
                syncSyncPoint(syncPoint);
            }
        }
        txId = getLastTxId();
    }
    LOG.debug("Syncing {} files", syncPointFiles.size());
    LOG.debug("Last transaction id {}", txId);
    SyncInfo syncInfo = new SyncInfo(syncPointFiles, false, txId);
    return syncInfo;
}