Examples with Scoped alluxio.util.interfaces.Scoped used on opensource projects

Example 1 with Scoped

use of alluxio.util.interfaces.Scoped in project alluxio by Alluxio.

the class InodeTree method syncPersistExistingDirectory.

/**
 * Synchronously persists an inode directory to the UFS. If concurrent calls are made, only
 * one thread will persist to UFS, and the others will wait until it is persisted.
 *
 * @param context journal context supplier
 * @param dir the inode directory to persist
 * @throws InvalidPathException if the path for the inode is invalid
 * @throws FileDoesNotExistException if the path for the inode is invalid
 */
public void syncPersistExistingDirectory(Supplier<JournalContext> context, InodeDirectoryView dir) throws IOException, InvalidPathException, FileDoesNotExistException {
    RetryPolicy retry = new ExponentialBackoffRetry(PERSIST_WAIT_BASE_SLEEP_MS, PERSIST_WAIT_MAX_SLEEP_MS, PERSIST_WAIT_MAX_RETRIES);
    while (retry.attempt()) {
        if (dir.getPersistenceState() == PersistenceState.PERSISTED) {
            // The directory is persisted
            return;
        }
        Optional<Scoped> persisting = mInodeLockManager.tryAcquirePersistingLock(dir.getId());
        if (!persisting.isPresent()) {
            // Someone else is doing this persist. Continue and wait for them to finish.
            continue;
        }
        try (Scoped s = persisting.get()) {
            if (dir.getPersistenceState() == PersistenceState.PERSISTED) {
                // The directory is persisted
                return;
            }
            mState.applyAndJournal(context, UpdateInodeEntry.newBuilder().setId(dir.getId()).setPersistenceState(PersistenceState.TO_BE_PERSISTED.name()).build());
            UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(dir.getId());
            syncPersistDirectory(dir).ifPresent(status -> {
                if (isRootId(dir.getId())) {
                    // Don't load the root dir metadata from UFS
                    return;
                }
                entry.setOwner(status.getOwner()).setGroup(status.getGroup()).setMode(status.getMode());
                Map<String, byte[]> xattr = status.getXAttr();
                if (xattr != null) {
                    entry.putAllXAttr(CommonUtils.convertToByteString(xattr));
                }
                Long lastModificationTime = status.getLastModifiedTime();
                if (lastModificationTime != null) {
                    entry.setLastModificationTimeMs(lastModificationTime).setOverwriteModificationTime(true);
                }
            });
            entry.setPersistenceState(PersistenceState.PERSISTED.name());
            mState.applyAndJournal(context, entry.build());
            return;
        }
    }
    throw new IOException(ExceptionMessage.FAILED_UFS_CREATE.getMessage(dir.getName()));
}

Example 2 with Scoped

use of alluxio.util.interfaces.Scoped in project alluxio by Alluxio.

the class InodeSyncStream method syncExistingInodeMetadata.

/**
 * Sync inode metadata with the UFS state.
 *
 * This method expects the {@code inodePath} to already exist in the inode tree.
 */
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
    if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
        throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
    }
    // Set to true if the given inode was deleted.
    boolean deletedInode = false;
    // whether we need to load metadata for the current path
    boolean loadMetadata = mLoadOnly;
    LOG.trace("Syncing inode metadata {}", inodePath.getUri());
    // The requested path already exists in Alluxio.
    Inode inode = inodePath.getInode();
    // initialize sync children to true if it is a listStatus call on a directory
    boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
    if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
        if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
            // Do not sync an incomplete file, since the UFS file is expected to not exist.
            return;
        }
        Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
        if (!persistingLock.isPresent()) {
            // written.
            return;
        }
        persistingLock.get().close();
        UfsStatus cachedStatus = null;
        boolean fileNotFound = false;
        try {
            cachedStatus = mStatusCache.getStatus(inodePath.getUri());
        } catch (FileNotFoundException e) {
            fileNotFound = true;
        }
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI ufsUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            String ufsFingerprint;
            Fingerprint ufsFpParsed;
            // When the status is not cached and it was not due to file not found, we retry
            if (fileNotFound) {
                ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else if (cachedStatus == null) {
                // TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
                ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else {
                // When the status is cached
                Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
                if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
                } else {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
                }
                ufsFingerprint = ufsFpParsed.serialize();
            }
            boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
            UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
            if (syncPlan.toUpdateMetaData()) {
                // It works by calling SetAttributeInternal on the inodePath.
                if (ufsFpParsed != null && ufsFpParsed.isValid()) {
                    short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
                    long opTimeMs = System.currentTimeMillis();
                    SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
                    String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
                    if (!owner.equals(Fingerprint.UNDERSCORE)) {
                        // Only set owner if not empty
                        builder.setOwner(owner);
                    }
                    String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
                    if (!group.equals(Fingerprint.UNDERSCORE)) {
                        // Only set group if not empty
                        builder.setGroup(group);
                    }
                    SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
                    mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
                }
            }
            if (syncPlan.toDelete()) {
                deletedInode = true;
                try {
                    // The options for deleting.
                    DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
                    mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
                } catch (DirectoryNotEmptyException | IOException e) {
                    // Should not happen, since it is an unchecked delete.
                    LOG.error("Unexpected error for unchecked delete.", e);
                }
            }
            if (syncPlan.toLoadMetadata()) {
                loadMetadata = true;
            }
            syncChildren = syncPlan.toSyncChildren();
        }
    }
    // if DescendantType.ONE we only sync children if we are syncing root of this stream
    if (mDescendantType == DescendantType.ONE) {
        syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
    }
    Map<String, Inode> inodeChildren = new HashMap<>();
    if (syncChildren) {
        // maps children name to inode
        mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
        // Fetch and populate children into the cache
        mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
        Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
        // Iterate over UFS listings and process UFS children.
        if (listStatus != null) {
            for (UfsStatus ufsChildStatus : listStatus) {
                if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
                    // Ufs child exists, but Alluxio child does not. Must load metadata.
                    loadMetadata = true;
                    break;
                }
            }
        }
    }
    // locked path
    if (deletedInode) {
        inodePath.removeLastInode();
    }
    // load metadata if necessary.
    if (loadMetadata && !skipLoad) {
        loadMetadataForPath(inodePath);
    }
    if (syncChildren) {
        // Iterate over Alluxio children and process persisted children.
        mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
            // its children.
            if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
                return;
            }
            // If we're performing a recursive sync, add each child of our current Inode to the queue
            AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
            mPendingPaths.add(child);
            // This asynchronously schedules a job to pre-fetch the statuses into the cache.
            if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
                mStatusCache.prefetchChildren(child, mMountTable);
            }
        });
    }
}

Example 3 with Scoped

use of alluxio.util.interfaces.Scoped in project alluxio by Alluxio.

the class AbstractPrimarySelectorTest method onStateChange.

@Test(timeout = TIMEOUT)
public void onStateChange() {
    AtomicInteger primaryCounter = new AtomicInteger(0);
    AtomicInteger standbyCounter = new AtomicInteger(0);
    Scoped listener = mSelector.onStateChange(state -> {
        if (state.equals(State.PRIMARY)) {
            primaryCounter.incrementAndGet();
        } else {
            standbyCounter.incrementAndGet();
        }
    });
    for (int i = 0; i < 10; i++) {
        mSelector.setState(State.PRIMARY);
        mSelector.setState(State.STANDBY);
    }
    assertEquals(10, primaryCounter.get());
    assertEquals(10, standbyCounter.get());
    listener.close();
    mSelector.setState(State.PRIMARY);
    mSelector.setState(State.STANDBY);
    assertEquals(10, primaryCounter.get());
    assertEquals(10, standbyCounter.get());
}

Example 4 with Scoped

use of alluxio.util.interfaces.Scoped in project alluxio by Alluxio.

the class FaultTolerantAlluxioMasterProcess method gainPrimacy.

/**
 * Upgrades the master to primary mode.
 *
 * If the master loses primacy during the journal upgrade, this method will clean up the partial
 * upgrade, leaving the master in standby mode.
 *
 * @return whether the master successfully upgraded to primary
 */
private boolean gainPrimacy() throws Exception {
    LOG.info("Becoming a leader.");
    // Don't upgrade if this master's primacy is unstable.
    AtomicBoolean unstable = new AtomicBoolean(false);
    try (Scoped scoped = mLeaderSelector.onStateChange(state -> unstable.set(true))) {
        if (mLeaderSelector.getState() != State.PRIMARY) {
            LOG.info("Lost leadership while becoming a leader.");
            unstable.set(true);
        }
        stopMasters();
        LOG.info("Standby stopped");
        try (Timer.Context ctx = MetricsSystem.timer(MetricKey.MASTER_JOURNAL_GAIN_PRIMACY_TIMER.getName()).time()) {
            mJournalSystem.gainPrimacy();
        }
        // We only check unstable here because mJournalSystem.gainPrimacy() is the only slow method
        if (unstable.get()) {
            LOG.info("Terminating an unstable attempt to become a leader.");
            if (ServerConfiguration.getBoolean(PropertyKey.MASTER_JOURNAL_EXIT_ON_DEMOTION)) {
                stop();
            } else {
                losePrimacy();
            }
            return false;
        }
    }
    try {
        startMasters(true);
    } catch (UnavailableException e) {
        LOG.warn("Error starting masters: {}", e.toString());
        stopMasters();
        return false;
    }
    mServingThread = new Thread(() -> {
        try {
            startCommonServices();
            startLeaderServing(" (gained leadership)", " (lost leadership)");
        } catch (Throwable t) {
            Throwable root = Throwables.getRootCause(t);
            if ((root != null && (root instanceof InterruptedException)) || Thread.interrupted()) {
                return;
            }
            ProcessUtils.fatalError(LOG, t, "Exception thrown in main serving thread");
        }
    }, "MasterServingThread");
    LOG.info("Starting a server thread.");
    mServingThread.start();
    if (!waitForReady(10 * Constants.MINUTE_MS)) {
        ThreadUtils.logAllThreads();
        throw new RuntimeException("Alluxio master failed to come up");
    }
    LOG.info("Primary started");
    return true;
}