Examples with FileTxnSnapLog org.apache.zookeeper.server.persistence.FileTxnSnapLog used on opensource projects

Example 41 with FileTxnSnapLog

use of org.apache.zookeeper.server.persistence.FileTxnSnapLog in project zookeeper by apache.

the class EmptiedSnapshotRecoveryTest method testRestoreWithTrustedEmptySnapFilesWhenFollowing.

@Test
public void testRestoreWithTrustedEmptySnapFilesWhenFollowing() throws Exception {
    QuorumUtil qu = new QuorumUtil(1);
    try {
        qu.startAll();
        String connString = qu.getConnectionStringForServer(1);
        try (ZooKeeper zk = new ZooKeeper(connString, CONNECTION_TIMEOUT, this)) {
            for (int i = 0; i < N_TRANSACTIONS; i++) {
                zk.create("/node-" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            }
        }
        int leaderIndex = qu.getLeaderServer();
        // Shut down the cluster and delete the snapshots from the followers
        for (int i = 1; i <= qu.ALL; i++) {
            qu.shutdown(i);
            if (i != leaderIndex) {
                FileTxnSnapLog txnLogFactory = qu.getPeer(i).peer.getTxnFactory();
                List<File> snapshots = txnLogFactory.findNRecentSnapshots(10);
                assertTrue(snapshots.size() > 0, "We have a snapshot to corrupt");
                for (File file : snapshots) {
                    Files.delete(file.toPath());
                }
                assertEquals(txnLogFactory.findNRecentSnapshots(10).size(), 0);
            }
        }
        // Start while trusting empty snapshots, verify that the followers save snapshots
        System.setProperty(FileTxnSnapLog.ZOOKEEPER_SNAPSHOT_TRUST_EMPTY, "true");
        qu.start(leaderIndex);
        for (int i = 1; i <= qu.ALL; i++) {
            if (i != leaderIndex) {
                qu.restart(i);
                FileTxnSnapLog txnLogFactory = qu.getPeer(i).peer.getTxnFactory();
                List<File> snapshots = txnLogFactory.findNRecentSnapshots(10);
                assertTrue(snapshots.size() > 0, "A snapshot should have been created on follower " + i);
            }
        }
        // Check that the created nodes are still there
        try (ZooKeeper zk = new ZooKeeper(connString, CONNECTION_TIMEOUT, this)) {
            for (int i = 0; i < N_TRANSACTIONS; i++) {
                assertNotNull(zk.exists("/node-" + i, false));
            }
        }
    } finally {
        System.clearProperty(FileTxnSnapLog.ZOOKEEPER_SNAPSHOT_TRUST_EMPTY);
        qu.tearDown();
    }
}

Example 42 with FileTxnSnapLog

use of org.apache.zookeeper.server.persistence.FileTxnSnapLog in project zookeeper by apache.

the class QuorumPeer method createFromConfig.

/**
 * Create a new QuorumPeer and apply all the values per the already-parsed config.
 *
 * @param config The appertained quorum peer config.
 * @return A QuorumPeer instantiated with specified peer config. Note this peer
 *         is not fully initialized; caller should finish initialization through
 *         additional configurations (connection factory settings, etc).
 *
 * @throws IOException
 */
public static QuorumPeer createFromConfig(QuorumPeerConfig config) throws IOException {
    QuorumPeer quorumPeer = new QuorumPeer();
    quorumPeer.setTxnFactory(new FileTxnSnapLog(config.getDataLogDir(), config.getDataDir()));
    quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled());
    quorumPeer.enableLocalSessionsUpgrading(config.isLocalSessionsUpgradingEnabled());
    quorumPeer.setElectionType(config.getElectionAlg());
    quorumPeer.setMyid(config.getServerId());
    quorumPeer.setTickTime(config.getTickTime());
    quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout());
    quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout());
    quorumPeer.setInitLimit(config.getInitLimit());
    quorumPeer.setSyncLimit(config.getSyncLimit());
    quorumPeer.setConnectToLearnerMasterLimit(config.getConnectToLearnerMasterLimit());
    quorumPeer.setObserverMasterPort(config.getObserverMasterPort());
    quorumPeer.setConfigFileName(config.getConfigFilename());
    quorumPeer.setClientPortListenBacklog(config.getClientPortListenBacklog());
    quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
    quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false);
    if (config.getLastSeenQuorumVerifier() != null) {
        quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false);
    }
    quorumPeer.initConfigInZKDatabase();
    quorumPeer.setSslQuorum(config.isSslQuorum());
    quorumPeer.setUsePortUnification(config.shouldUsePortUnification());
    quorumPeer.setLearnerType(config.getPeerType());
    quorumPeer.setSyncEnabled(config.getSyncEnabled());
    quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs());
    if (config.sslQuorumReloadCertFiles) {
        quorumPeer.getX509Util().enableCertFileReloading();
    }
    quorumPeer.setMultiAddressEnabled(config.isMultiAddressEnabled());
    quorumPeer.setMultiAddressReachabilityCheckEnabled(config.isMultiAddressReachabilityCheckEnabled());
    quorumPeer.setMultiAddressReachabilityCheckTimeoutMs(config.getMultiAddressReachabilityCheckTimeoutMs());
    // sets quorum sasl authentication configurations
    quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl);
    if (quorumPeer.isQuorumSaslAuthEnabled()) {
        quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl);
        quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl);
        quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal);
        quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext);
        quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext);
    }
    quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize);
    if (config.jvmPauseMonitorToRun) {
        quorumPeer.setJvmPauseMonitor(new JvmPauseMonitor(config));
    }
    return quorumPeer;
}

Example 43 with FileTxnSnapLog

use of org.apache.zookeeper.server.persistence.FileTxnSnapLog in project zookeeper by apache.

the class QuorumPeerMain method runFromConfig.

public void runFromConfig(QuorumPeerConfig config) throws IOException, AdminServerException {
    try {
        ManagedUtil.registerLog4jMBeans();
    } catch (JMException e) {
        LOG.warn("Unable to register log4j JMX control", e);
    }
    LOG.info("Starting quorum peer, myid=" + config.getServerId());
    final MetricsProvider metricsProvider;
    try {
        metricsProvider = MetricsProviderBootstrap.startMetricsProvider(config.getMetricsProviderClassName(), config.getMetricsProviderConfiguration());
    } catch (MetricsProviderLifeCycleException error) {
        throw new IOException("Cannot boot MetricsProvider " + config.getMetricsProviderClassName(), error);
    }
    try {
        ServerMetrics.metricsProviderInitialized(metricsProvider);
        ProviderRegistry.initialize();
        ServerCnxnFactory cnxnFactory = null;
        ServerCnxnFactory secureCnxnFactory = null;
        if (config.getClientPortAddress() != null) {
            cnxnFactory = ServerCnxnFactory.createFactory();
            cnxnFactory.configure(config.getClientPortAddress(), config.getMaxClientCnxns(), config.getClientPortListenBacklog(), false);
        }
        if (config.getSecureClientPortAddress() != null) {
            secureCnxnFactory = ServerCnxnFactory.createFactory();
            secureCnxnFactory.configure(config.getSecureClientPortAddress(), config.getMaxClientCnxns(), config.getClientPortListenBacklog(), true);
        }
        quorumPeer = getQuorumPeer();
        quorumPeer.setTxnFactory(new FileTxnSnapLog(config.getDataLogDir(), config.getDataDir()));
        quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled());
        quorumPeer.enableLocalSessionsUpgrading(config.isLocalSessionsUpgradingEnabled());
        // quorumPeer.setQuorumPeers(config.getAllMembers());
        quorumPeer.setElectionType(config.getElectionAlg());
        quorumPeer.setMyid(config.getServerId());
        quorumPeer.setTickTime(config.getTickTime());
        quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout());
        quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout());
        quorumPeer.setInitLimit(config.getInitLimit());
        quorumPeer.setSyncLimit(config.getSyncLimit());
        quorumPeer.setConnectToLearnerMasterLimit(config.getConnectToLearnerMasterLimit());
        quorumPeer.setObserverMasterPort(config.getObserverMasterPort());
        quorumPeer.setConfigFileName(config.getConfigFilename());
        quorumPeer.setClientPortListenBacklog(config.getClientPortListenBacklog());
        quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
        quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false);
        if (config.getLastSeenQuorumVerifier() != null) {
            quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false);
        }
        quorumPeer.initConfigInZKDatabase();
        quorumPeer.setCnxnFactory(cnxnFactory);
        quorumPeer.setSecureCnxnFactory(secureCnxnFactory);
        quorumPeer.setSslQuorum(config.isSslQuorum());
        quorumPeer.setUsePortUnification(config.shouldUsePortUnification());
        quorumPeer.setLearnerType(config.getPeerType());
        quorumPeer.setSyncEnabled(config.getSyncEnabled());
        quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs());
        if (config.sslQuorumReloadCertFiles) {
            quorumPeer.getX509Util().enableCertFileReloading();
        }
        quorumPeer.setMultiAddressEnabled(config.isMultiAddressEnabled());
        quorumPeer.setMultiAddressReachabilityCheckEnabled(config.isMultiAddressReachabilityCheckEnabled());
        quorumPeer.setMultiAddressReachabilityCheckTimeoutMs(config.getMultiAddressReachabilityCheckTimeoutMs());
        // sets quorum sasl authentication configurations
        quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl);
        if (quorumPeer.isQuorumSaslAuthEnabled()) {
            quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl);
            quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl);
            quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal);
            quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext);
            quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext);
        }
        quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize);
        quorumPeer.initialize();
        if (config.jvmPauseMonitorToRun) {
            quorumPeer.setJvmPauseMonitor(new JvmPauseMonitor(config));
        }
        quorumPeer.start();
        ZKAuditProvider.addZKStartStopAuditLog();
        quorumPeer.join();
    } catch (InterruptedException e) {
        // warn, but generally this is ok
        LOG.warn("Quorum Peer interrupted", e);
    } finally {
        try {
            metricsProvider.stop();
        } catch (Throwable error) {
            LOG.warn("Error while stopping metrics", error);
        }
    }
}

Example 44 with FileTxnSnapLog

use of org.apache.zookeeper.server.persistence.FileTxnSnapLog in project zookeeper by apache.

the class Zab1_0Test method createFollower.

private ConversableFollower createFollower(File tmpDir, QuorumPeer peer) throws IOException {
    FileTxnSnapLog logFactory = new FileTxnSnapLog(tmpDir, tmpDir);
    peer.setTxnFactory(logFactory);
    ZKDatabase zkDb = new ZKDatabase(logFactory);
    FollowerZooKeeperServer zk = new FollowerZooKeeperServer(logFactory, peer, zkDb);
    peer.setZKDatabase(zkDb);
    return new ConversableFollower(peer, zk);
}

Example 45 with FileTxnSnapLog

use of org.apache.zookeeper.server.persistence.FileTxnSnapLog in project zookeeper by apache.

the class Zab1_0Test method testPopulatedLeaderConversation.

public void testPopulatedLeaderConversation(PopulatedLeaderConversation conversation, int ops) throws Exception {
    Socket[] pair = getSocketPair();
    Socket leaderSocket = pair[0];
    Socket followerSocket = pair[1];
    File tmpDir = File.createTempFile("test", "dir", testData);
    tmpDir.delete();
    tmpDir.mkdir();
    LeadThread leadThread = null;
    Leader leader = null;
    try {
        // Setup a database with two znodes
        FileTxnSnapLog snapLog = new FileTxnSnapLog(tmpDir, tmpDir);
        ZKDatabase zkDb = new ZKDatabase(snapLog);
        assertTrue(ops >= 1);
        long zxid = ZxidUtils.makeZxid(1, 0);
        for (int i = 1; i <= ops; i++) {
            zxid = ZxidUtils.makeZxid(1, i);
            String path = "/foo-" + i;
            zkDb.processTxn(new TxnHeader(13, 1000 + i, zxid, 30 + i, ZooDefs.OpCode.create), new CreateTxn(path, "fpjwasalsohere".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 1), null);
            Stat stat = new Stat();
            assertEquals("fpjwasalsohere", new String(zkDb.getData(path, stat, null)));
        }
        assertTrue(zxid > ZxidUtils.makeZxid(1, 0));
        // Generate snapshot and close files.
        snapLog.save(zkDb.getDataTree(), zkDb.getSessionWithTimeOuts(), false);
        snapLog.close();
        QuorumPeer peer = createQuorumPeer(tmpDir);
        leader = createLeader(tmpDir, peer);
        peer.leader = leader;
        // Set the last accepted epoch and current epochs to be 1
        peer.setAcceptedEpoch(1);
        peer.setCurrentEpoch(1);
        leadThread = new LeadThread(leader);
        leadThread.start();
        while (leader.cnxAcceptor == null || !leader.cnxAcceptor.isAlive()) {
            Thread.sleep(20);
        }
        LearnerHandler lh = new LearnerHandler(leaderSocket, new BufferedInputStream(leaderSocket.getInputStream()), leader);
        lh.start();
        leaderSocket.setSoTimeout(4000);
        InputArchive ia = BinaryInputArchive.getArchive(followerSocket.getInputStream());
        OutputArchive oa = BinaryOutputArchive.getArchive(followerSocket.getOutputStream());
        conversation.converseWithLeader(ia, oa, leader, zxid);
    } finally {
        if (leader != null) {
            leader.shutdown("end of test");
        }
        if (leadThread != null) {
            leadThread.interrupt();
            leadThread.join();
        }
        TestUtils.deleteFileRecursively(tmpDir);
    }
}