Examples with JournalContext alluxio.master.journal.JournalContext used on opensource projects

Example 21 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method writeJournal.

@Test
public void writeJournal() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context.
    final int entryCount = 10;
    try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
        for (int i = 0; i < entryCount; i++) {
            journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
        }
    }
    // Wait for sequences to be caught up.
    CommonUtils.waitFor("full state acquired", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
}

Example 22 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class RaftJournalTest method gainPrimacyDuringCatchup.

@Test
public void gainPrimacyDuringCatchup() throws Exception {
    // Create a counting master implementation that counts how many journal entries it processed.
    CountingDummyFileSystemMaster countingMaster = new CountingDummyFileSystemMaster();
    mFollowerJournalSystem.createJournal(countingMaster);
    // Using a large entry count for catching transition while in-progress.
    final int entryCount = 100;
    // Suspend follower journal system.
    mFollowerJournalSystem.suspend(null);
    // Create entries on the leader journal context.
    // These will be replicated to follower journal context once resumed.
    ForkJoinPool.commonPool().submit(() -> {
        try (JournalContext journalContext = mLeaderJournalSystem.createJournal(new NoopMaster()).createJournalContext()) {
            for (int i = 0; i < entryCount; i++) {
                journalContext.append(alluxio.proto.journal.Journal.JournalEntry.newBuilder().setInodeLastModificationTime(File.InodeLastModificationTimeEntry.newBuilder().setId(i).build()).build());
            }
        } catch (Exception e) {
            Assert.fail(String.format("Failed while writing entries: %s", e.toString()));
        }
    }).get();
    // Catch up follower journal to a large index to be able to transition while in progress.
    Map<String, Long> backupSequences = new HashMap<>();
    backupSequences.put("FileSystemMaster", (long) entryCount);
    // Set delay for each internal processing of entries before initiating catch-up.
    countingMaster.setApplyDelay(100);
    CatchupFuture catchupFuture = mFollowerJournalSystem.catchup(backupSequences);
    // Wait until advancing starts.
    CommonUtils.waitFor("Advancing to start.", () -> countingMaster.getApplyCount() > 0, mWaitOptions);
    // Gain primacy in follower journal
    promoteFollower();
    // Validate it catches up.
    CommonUtils.waitFor("Old olf follower to catch up.", () -> countingMaster.getApplyCount() == entryCount, mWaitOptions);
    // Follower should no longer be suspended after becoming primary.
    Assert.assertFalse(mFollowerJournalSystem.isSuspended());
}

Example 23 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class TransformManager method execute.

/**
 * Executes the plans for the table transformation.
 *
 * This method executes a transformation job with type{@link CompositeConfig},
 * the transformation job concurrently executes the plans,
 * each plan has a list of jobs to be executed sequentially.
 *
 * This method triggers the execution of the transformation job asynchronously without waiting
 * for it to finish. The returned job ID can be used to poll the job service for the status of
 * this transformation.
 *
 * @param dbName the database name
 * @param tableName the table name
 * @param definition the parsed transformation definition
 * @return the job ID for the transformation job
 * @throws IOException when there is an ongoing transformation on the table, or the transformation
 *    job fails to be started, or all partitions of the table have been transformed with the same
 *    definition
 */
public long execute(String dbName, String tableName, TransformDefinition definition) throws IOException {
    List<TransformPlan> plans = mCatalog.getTransformPlan(dbName, tableName, definition);
    if (plans.isEmpty()) {
        throw new IOException(ExceptionMessage.TABLE_ALREADY_TRANSFORMED.getMessage(dbName, tableName, definition.getDefinition()));
    }
    Pair<String, String> dbTable = new Pair<>(dbName, tableName);
    // Atomically try to acquire the permit to execute the transformation job.
    // This PUT does not need to be journaled, because if this PUT succeeds and master crashes,
    // when master restarts, this temporary placeholder entry will not exist, which is correct
    // behavior.
    Long existingJobId = mState.acquireJobPermit(dbTable);
    if (existingJobId != null) {
        if (existingJobId == INVALID_JOB_ID) {
            throw new IOException("A concurrent transformation request is going to be executed");
        } else {
            throw new IOException(ExceptionMessage.TABLE_BEING_TRANSFORMED.getMessage(existingJobId.toString(), tableName, dbName));
        }
    }
    ArrayList<JobConfig> concurrentJobs = new ArrayList<>(plans.size());
    for (TransformPlan plan : plans) {
        concurrentJobs.add(new CompositeConfig(plan.getJobConfigs(), true));
    }
    CompositeConfig transformJob = new CompositeConfig(concurrentJobs, false);
    long jobId;
    try {
        jobId = mJobMasterClient.run(transformJob);
    } catch (IOException e) {
        // The job fails to start, clear the acquired permit for execution.
        // No need to journal this REMOVE, if master crashes, when it restarts, the permit placeholder
        // entry will not exist any more, which is correct behavior.
        mState.releaseJobPermit(dbTable);
        String error = String.format("Fails to start job to transform table %s in database %s", tableName, dbName);
        LOG.error(error, e);
        throw new IOException(error, e);
    }
    Map<String, Layout> transformedLayouts = new HashMap<>(plans.size());
    for (TransformPlan plan : plans) {
        transformedLayouts.put(plan.getBaseLayout().getSpec(), plan.getTransformedLayout());
    }
    AddTransformJobInfoEntry journalEntry = AddTransformJobInfoEntry.newBuilder().setDbName(dbName).setTableName(tableName).setDefinition(definition.getDefinition()).setJobId(jobId).putAllTransformedLayouts(Maps.transformValues(transformedLayouts, Layout::toProto)).build();
    try (JournalContext journalContext = mCreateJournalContext.apply()) {
        applyAndJournal(journalContext, Journal.JournalEntry.newBuilder().setAddTransformJobInfo(journalEntry).build());
    }
    return jobId;
}

Example 24 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class AccessTimeUpdaterTest method updateAccessTimePrecision.

@Test
public void updateAccessTimePrecision() throws Exception {
    mAccessTimeUpdater = new AccessTimeUpdater(mFileSystemMaster, mInodeTree, mContext.getJournalSystem(), 0, Constants.HOUR_MS, 0);
    mAccessTimeUpdater.start();
    String path = "/foo";
    createInode(path, CreateFileContext.defaults());
    JournalContext journalContext = mock(JournalContext.class);
    when(journalContext.get()).thenReturn(journalContext);
    when(mFileSystemMaster.createJournalContext()).thenReturn(journalContext);
    long accessTime = CommonUtils.getCurrentMs() + 100L;
    long inodeId;
    try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), InodeTree.LockPattern.READ)) {
        mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), accessTime);
        inodeId = lockedInodes.getInode().getId();
    }
    // verify inode attribute is not updated
    assertNotEquals(accessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs());
    // verify journal entry is not logged yet
    verify(journalContext, never()).append(any(Journal.JournalEntry.class));
    long newAccessTime = CommonUtils.getCurrentMs() + 2 * Constants.HOUR_MS;
    // update access time with a much later timestamp
    try (LockedInodePath lockedInodes = mInodeTree.lockFullInodePath(new AlluxioURI(path), InodeTree.LockPattern.READ)) {
        mAccessTimeUpdater.updateAccessTime(journalContext, lockedInodes.getInode(), newAccessTime);
        inodeId = lockedInodes.getInode().getId();
    }
    // verify inode attribute is updated
    assertEquals(newAccessTime, mInodeStore.get(inodeId).get().getLastAccessTimeMs());
    // / verify journal entry is logged
    ArgumentCaptor<Journal.JournalEntry> captor = ArgumentCaptor.forClass(Journal.JournalEntry.class);
    verify(journalContext).append(captor.capture());
    assertTrue(captor.getValue().hasUpdateInode());
    assertEquals(inodeId, captor.getValue().getUpdateInode().getId());
    assertEquals(newAccessTime, captor.getValue().getUpdateInode().getLastAccessTimeMs());
}

Example 25 with JournalContext

use of alluxio.master.journal.JournalContext in project alluxio by Alluxio.

the class DefaultFileSystemMaster method start.

@Override
public void start(Boolean isPrimary) throws IOException {
    super.start(isPrimary);
    if (isPrimary) {
        LOG.info("Starting fs master as primary");
        InodeDirectory root = mInodeTree.getRoot();
        if (root == null) {
            try (JournalContext context = createJournalContext()) {
                mInodeTree.initializeRoot(SecurityUtils.getOwner(mMasterContext.getUserState()), SecurityUtils.getGroup(mMasterContext.getUserState(), ServerConfiguration.global()), ModeUtils.applyDirectoryUMask(Mode.createFullAccess(), ServerConfiguration.getString(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)), context);
            }
        } else if (!ServerConfiguration.getBoolean(PropertyKey.MASTER_SKIP_ROOT_ACL_CHECK)) {
            // For backwards-compatibility:
            // Empty root owner indicates that previously the master had no security. In this case, the
            // master is allowed to be started with security turned on.
            String serverOwner = SecurityUtils.getOwner(mMasterContext.getUserState());
            if (SecurityUtils.isSecurityEnabled(ServerConfiguration.global()) && !root.getOwner().isEmpty() && !root.getOwner().equals(serverOwner)) {
                // user is not the previous owner
                throw new PermissionDeniedException(ExceptionMessage.PERMISSION_DENIED.getMessage(String.format("Unauthorized user on root. inode owner: %s current user: %s", root.getOwner(), serverOwner)));
            }
        }
        // Initialize the ufs manager from the mount table.
        for (String key : mMountTable.getMountTable().keySet()) {
            if (key.equals(MountTable.ROOT)) {
                continue;
            }
            MountInfo mountInfo = mMountTable.getMountTable().get(key);
            UnderFileSystemConfiguration ufsConf = UnderFileSystemConfiguration.defaults(ServerConfiguration.global()).createMountSpecificConf(mountInfo.getOptions().getPropertiesMap()).setReadOnly(mountInfo.getOptions().getReadOnly()).setShared(mountInfo.getOptions().getShared());
            mUfsManager.addMount(mountInfo.getMountId(), mountInfo.getUfsUri(), ufsConf);
        }
        // Startup Checks and Periodic Threads.
        // Rebuild the list of persist jobs (mPersistJobs) and map of pending persist requests
        // (mPersistRequests)
        long persistInitialIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_INITIAL_INTERVAL_MS);
        long persistMaxIntervalMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_INTERVAL_MS);
        long persistMaxWaitMs = ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_MAX_TOTAL_WAIT_TIME_MS);
        for (Long id : mInodeTree.getToBePersistedIds()) {
            Inode inode = mInodeStore.get(id).get();
            if (inode.isDirectory() || // When file is completed it is added to persist reqs
            !inode.asFile().isCompleted() || inode.getPersistenceState() != PersistenceState.TO_BE_PERSISTED || inode.asFile().getShouldPersistTime() == Constants.NO_AUTO_PERSIST) {
                continue;
            }
            InodeFile inodeFile = inode.asFile();
            if (inodeFile.getPersistJobId() == Constants.PERSISTENCE_INVALID_JOB_ID) {
                mPersistRequests.put(inodeFile.getId(), new alluxio.time.ExponentialTimer(persistInitialIntervalMs, persistMaxIntervalMs, getPersistenceWaitTime(inodeFile.getShouldPersistTime()), persistMaxWaitMs));
            } else {
                AlluxioURI path;
                try {
                    path = mInodeTree.getPath(inodeFile);
                } catch (FileDoesNotExistException e) {
                    LOG.error("Failed to determine path for inode with id {}", id, e);
                    continue;
                }
                addPersistJob(id, inodeFile.getPersistJobId(), getPersistenceWaitTime(inodeFile.getShouldPersistTime()), path, inodeFile.getTempUfsPath());
            }
        }
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED)) {
            validateInodeBlocks(true);
        }
        int blockIntegrityCheckInterval = (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERIODIC_BLOCK_INTEGRITY_CHECK_INTERVAL);
        if (blockIntegrityCheckInterval > 0) {
            // negative or zero interval implies disabled
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_BLOCK_INTEGRITY_CHECK, new BlockIntegrityChecker(this), blockIntegrityCheckInterval, ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_TTL_CHECK, new InodeTtlChecker(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_LOST_FILES_DETECTION, new LostFileDetector(this, mInodeTree), (int) ServerConfiguration.getMs(PropertyKey.MASTER_LOST_WORKER_FILE_DETECTION_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        mReplicationCheckHeartbeatThread = new HeartbeatThread(HeartbeatContext.MASTER_REPLICATION_CHECK, new alluxio.master.file.replication.ReplicationChecker(mInodeTree, mBlockMaster, mSafeModeManager, mJobMasterClientPool), (int) ServerConfiguration.getMs(PropertyKey.MASTER_REPLICATION_CHECK_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState());
        ReconfigurableRegistry.register(this);
        getExecutorService().submit(mReplicationCheckHeartbeatThread);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER, new PersistenceScheduler(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        mPersistCheckerPool = new java.util.concurrent.ThreadPoolExecutor(PERSIST_CHECKER_POOL_THREADS, PERSIST_CHECKER_POOL_THREADS, 1, java.util.concurrent.TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), alluxio.util.ThreadFactoryUtils.build("Persist-Checker-%d", true));
        mPersistCheckerPool.allowCoreThreadTimeOut(true);
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_PERSISTENCE_CHECKER, new PersistenceChecker(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS), ServerConfiguration.global(), mMasterContext.getUserState()));
        getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_TIME_SERIES, new TimeSeriesRecorder(), (int) ServerConfiguration.getMs(PropertyKey.MASTER_METRICS_TIME_SERIES_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        if (ServerConfiguration.getBoolean(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED)) {
            mAsyncAuditLogWriter = new AsyncUserAccessAuditLogWriter("AUDIT_LOG");
            mAsyncAuditLogWriter.start();
            MetricsSystem.registerGaugeIfAbsent(MetricKey.MASTER_AUDIT_LOG_ENTRIES_SIZE.getName(), () -> mAsyncAuditLogWriter != null ? mAsyncAuditLogWriter.getAuditLogEntriesSize() : -1);
        }
        if (ServerConfiguration.getBoolean(PropertyKey.UNDERFS_CLEANUP_ENABLED)) {
            getExecutorService().submit(new HeartbeatThread(HeartbeatContext.MASTER_UFS_CLEANUP, new UfsCleaner(this), (int) ServerConfiguration.getMs(PropertyKey.UNDERFS_CLEANUP_INTERVAL), ServerConfiguration.global(), mMasterContext.getUserState()));
        }
        mAccessTimeUpdater.start();
        mSyncManager.start();
    }
}