Examples with ZKDatabase org.apache.zookeeper.server.ZKDatabase used on opensource projects

Example 21 with ZKDatabase

use of org.apache.zookeeper.server.ZKDatabase in project zookeeper by apache.

the class FollowerResyncConcurrencyTest method waitForSync.

/**
 * Wait for all server to have the same lastProccessedZxid. Timeout in seconds
 */
private boolean waitForSync(QuorumUtil qu, int index, int timeout) throws InterruptedException {
    LOG.info("Wait for server to sync");
    int leaderIndex = (index == 1) ? 2 : 1;
    ZKDatabase restartedDb = qu.getPeer(index).peer.getActiveServer().getZKDatabase();
    ZKDatabase cleanDb = qu.getPeer(3).peer.getActiveServer().getZKDatabase();
    ZKDatabase leadDb = qu.getPeer(leaderIndex).peer.getActiveServer().getZKDatabase();
    long leadZxid = 0;
    long cleanZxid = 0;
    long restartedZxid = 0;
    for (int i = 0; i < timeout; ++i) {
        leadZxid = leadDb.getDataTreeLastProcessedZxid();
        cleanZxid = cleanDb.getDataTreeLastProcessedZxid();
        restartedZxid = restartedDb.getDataTreeLastProcessedZxid();
        if (leadZxid == cleanZxid && leadZxid == restartedZxid) {
            return true;
        }
        Thread.sleep(1000);
    }
    LOG.info("Timeout waiting for zxid to sync: leader 0x{} clean 0x{} restarted 0x{}", Long.toHexString(leadZxid), Long.toHexString(cleanZxid), Long.toHexString(restartedZxid));
    return false;
}

Example 22 with ZKDatabase

use of org.apache.zookeeper.server.ZKDatabase in project zookeeper by apache.

the class GetProposalFromTxnTest method testGetProposalFromTxn.

/**
 * Test loading proposal from txnlog
 *
 * @throws Exception
 *             an exception might be thrown here
 */
@Test
public void testGetProposalFromTxn() throws Exception {
    File tmpDir = ClientBase.createTmpDir();
    ClientBase.setupTestEnv();
    ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
    SyncRequestProcessor.setSnapCount(100);
    final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
    ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
    f.startup(zks);
    assertTrue(ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT), "waiting for server being up ");
    ZooKeeper zk = ClientBase.createZKClient(HOSTPORT);
    // Generate transaction so we will have some txnlog
    Long[] zxids = new Long[MSG_COUNT];
    try {
        String data = "data";
        byte[] bytes = data.getBytes();
        for (int i = 0; i < MSG_COUNT; i++) {
            Stat stat = new Stat();
            zk.create("/invalidsnap-" + i, bytes, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            zk.getData("/invalidsnap-" + i, null, stat);
            zxids[i] = stat.getCzxid();
        }
    } finally {
        zk.close();
    }
    // shutdown and start zookeeper again
    f.shutdown();
    zks.shutdown();
    assertTrue(ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT), "waiting for server to shutdown");
    zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
    zks.startdata();
    ZKDatabase db = zks.getZKDatabase();
    // Set sizeLimit to be very high number, so we can pull all transactions
    // from txnlog
    Iterator<Proposal> itr = db.getProposalsFromTxnLog(zxids[0], 10000000);
    int createCount = 0;
    ArrayList<Long> retrievedZxids = new ArrayList<Long>(MSG_COUNT);
    // Get zxid of create requests
    while (itr.hasNext()) {
        Proposal proposal = itr.next();
        TxnLogEntry logEntry = SerializeUtils.deserializeTxn(proposal.packet.getData());
        TxnHeader hdr = logEntry.getHeader();
        Record rec = logEntry.getTxn();
        if (hdr.getType() == OpCode.create) {
            retrievedZxids.add(hdr.getZxid());
            createCount++;
        }
    }
    // All zxid should match what we created
    assertTrue(Arrays.equals(zxids, retrievedZxids.toArray(new Long[0])), "Zxids missmatches");
    // There should be 2000 create requests
    assertTrue((createCount == MSG_COUNT), "create proposal count == " + MSG_COUNT);
    // We are requesting half the number of transaction from the snapshot
    // this should exceed threshold (ZKDatabase.snapshotSizeFactor)
    db.setSnapshotSizeFactor(0.33);
    long sizeLimit = db.calculateTxnLogSizeLimit();
    itr = db.getProposalsFromTxnLog(zxids[MSG_COUNT / 2], sizeLimit);
    assertFalse((itr.hasNext()), "Expect empty proposal");
    f.shutdown();
    zks.shutdown();
}

Example 23 with ZKDatabase

use of org.apache.zookeeper.server.ZKDatabase in project zookeeper by apache.

the class QuorumPeer method createFromConfig.

/**
 * Create a new QuorumPeer and apply all the values per the already-parsed config.
 *
 * @param config The appertained quorum peer config.
 * @return A QuorumPeer instantiated with specified peer config. Note this peer
 *         is not fully initialized; caller should finish initialization through
 *         additional configurations (connection factory settings, etc).
 *
 * @throws IOException
 */
public static QuorumPeer createFromConfig(QuorumPeerConfig config) throws IOException {
    QuorumPeer quorumPeer = new QuorumPeer();
    quorumPeer.setTxnFactory(new FileTxnSnapLog(config.getDataLogDir(), config.getDataDir()));
    quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled());
    quorumPeer.enableLocalSessionsUpgrading(config.isLocalSessionsUpgradingEnabled());
    quorumPeer.setElectionType(config.getElectionAlg());
    quorumPeer.setMyid(config.getServerId());
    quorumPeer.setTickTime(config.getTickTime());
    quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout());
    quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout());
    quorumPeer.setInitLimit(config.getInitLimit());
    quorumPeer.setSyncLimit(config.getSyncLimit());
    quorumPeer.setConnectToLearnerMasterLimit(config.getConnectToLearnerMasterLimit());
    quorumPeer.setObserverMasterPort(config.getObserverMasterPort());
    quorumPeer.setConfigFileName(config.getConfigFilename());
    quorumPeer.setClientPortListenBacklog(config.getClientPortListenBacklog());
    quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
    quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false);
    if (config.getLastSeenQuorumVerifier() != null) {
        quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false);
    }
    quorumPeer.initConfigInZKDatabase();
    quorumPeer.setSslQuorum(config.isSslQuorum());
    quorumPeer.setUsePortUnification(config.shouldUsePortUnification());
    quorumPeer.setLearnerType(config.getPeerType());
    quorumPeer.setSyncEnabled(config.getSyncEnabled());
    quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs());
    if (config.sslQuorumReloadCertFiles) {
        quorumPeer.getX509Util().enableCertFileReloading();
    }
    quorumPeer.setMultiAddressEnabled(config.isMultiAddressEnabled());
    quorumPeer.setMultiAddressReachabilityCheckEnabled(config.isMultiAddressReachabilityCheckEnabled());
    quorumPeer.setMultiAddressReachabilityCheckTimeoutMs(config.getMultiAddressReachabilityCheckTimeoutMs());
    // sets quorum sasl authentication configurations
    quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl);
    if (quorumPeer.isQuorumSaslAuthEnabled()) {
        quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl);
        quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl);
        quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal);
        quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext);
        quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext);
    }
    quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize);
    if (config.jvmPauseMonitorToRun) {
        quorumPeer.setJvmPauseMonitor(new JvmPauseMonitor(config));
    }
    return quorumPeer;
}

Example 24 with ZKDatabase

use of org.apache.zookeeper.server.ZKDatabase in project zookeeper by apache.

the class QuorumPeerMain method runFromConfig.

public void runFromConfig(QuorumPeerConfig config) throws IOException, AdminServerException {
    try {
        ManagedUtil.registerLog4jMBeans();
    } catch (JMException e) {
        LOG.warn("Unable to register log4j JMX control", e);
    }
    LOG.info("Starting quorum peer, myid=" + config.getServerId());
    final MetricsProvider metricsProvider;
    try {
        metricsProvider = MetricsProviderBootstrap.startMetricsProvider(config.getMetricsProviderClassName(), config.getMetricsProviderConfiguration());
    } catch (MetricsProviderLifeCycleException error) {
        throw new IOException("Cannot boot MetricsProvider " + config.getMetricsProviderClassName(), error);
    }
    try {
        ServerMetrics.metricsProviderInitialized(metricsProvider);
        ProviderRegistry.initialize();
        ServerCnxnFactory cnxnFactory = null;
        ServerCnxnFactory secureCnxnFactory = null;
        if (config.getClientPortAddress() != null) {
            cnxnFactory = ServerCnxnFactory.createFactory();
            cnxnFactory.configure(config.getClientPortAddress(), config.getMaxClientCnxns(), config.getClientPortListenBacklog(), false);
        }
        if (config.getSecureClientPortAddress() != null) {
            secureCnxnFactory = ServerCnxnFactory.createFactory();
            secureCnxnFactory.configure(config.getSecureClientPortAddress(), config.getMaxClientCnxns(), config.getClientPortListenBacklog(), true);
        }
        quorumPeer = getQuorumPeer();
        quorumPeer.setTxnFactory(new FileTxnSnapLog(config.getDataLogDir(), config.getDataDir()));
        quorumPeer.enableLocalSessions(config.areLocalSessionsEnabled());
        quorumPeer.enableLocalSessionsUpgrading(config.isLocalSessionsUpgradingEnabled());
        // quorumPeer.setQuorumPeers(config.getAllMembers());
        quorumPeer.setElectionType(config.getElectionAlg());
        quorumPeer.setMyid(config.getServerId());
        quorumPeer.setTickTime(config.getTickTime());
        quorumPeer.setMinSessionTimeout(config.getMinSessionTimeout());
        quorumPeer.setMaxSessionTimeout(config.getMaxSessionTimeout());
        quorumPeer.setInitLimit(config.getInitLimit());
        quorumPeer.setSyncLimit(config.getSyncLimit());
        quorumPeer.setConnectToLearnerMasterLimit(config.getConnectToLearnerMasterLimit());
        quorumPeer.setObserverMasterPort(config.getObserverMasterPort());
        quorumPeer.setConfigFileName(config.getConfigFilename());
        quorumPeer.setClientPortListenBacklog(config.getClientPortListenBacklog());
        quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
        quorumPeer.setQuorumVerifier(config.getQuorumVerifier(), false);
        if (config.getLastSeenQuorumVerifier() != null) {
            quorumPeer.setLastSeenQuorumVerifier(config.getLastSeenQuorumVerifier(), false);
        }
        quorumPeer.initConfigInZKDatabase();
        quorumPeer.setCnxnFactory(cnxnFactory);
        quorumPeer.setSecureCnxnFactory(secureCnxnFactory);
        quorumPeer.setSslQuorum(config.isSslQuorum());
        quorumPeer.setUsePortUnification(config.shouldUsePortUnification());
        quorumPeer.setLearnerType(config.getPeerType());
        quorumPeer.setSyncEnabled(config.getSyncEnabled());
        quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs());
        if (config.sslQuorumReloadCertFiles) {
            quorumPeer.getX509Util().enableCertFileReloading();
        }
        quorumPeer.setMultiAddressEnabled(config.isMultiAddressEnabled());
        quorumPeer.setMultiAddressReachabilityCheckEnabled(config.isMultiAddressReachabilityCheckEnabled());
        quorumPeer.setMultiAddressReachabilityCheckTimeoutMs(config.getMultiAddressReachabilityCheckTimeoutMs());
        // sets quorum sasl authentication configurations
        quorumPeer.setQuorumSaslEnabled(config.quorumEnableSasl);
        if (quorumPeer.isQuorumSaslAuthEnabled()) {
            quorumPeer.setQuorumServerSaslRequired(config.quorumServerRequireSasl);
            quorumPeer.setQuorumLearnerSaslRequired(config.quorumLearnerRequireSasl);
            quorumPeer.setQuorumServicePrincipal(config.quorumServicePrincipal);
            quorumPeer.setQuorumServerLoginContext(config.quorumServerLoginContext);
            quorumPeer.setQuorumLearnerLoginContext(config.quorumLearnerLoginContext);
        }
        quorumPeer.setQuorumCnxnThreadsSize(config.quorumCnxnThreadsSize);
        quorumPeer.initialize();
        if (config.jvmPauseMonitorToRun) {
            quorumPeer.setJvmPauseMonitor(new JvmPauseMonitor(config));
        }
        quorumPeer.start();
        ZKAuditProvider.addZKStartStopAuditLog();
        quorumPeer.join();
    } catch (InterruptedException e) {
        // warn, but generally this is ok
        LOG.warn("Quorum Peer interrupted", e);
    } finally {
        try {
            metricsProvider.stop();
        } catch (Throwable error) {
            LOG.warn("Error while stopping metrics", error);
        }
    }
}

Example 25 with ZKDatabase

use of org.apache.zookeeper.server.ZKDatabase in project zookeeper by apache.

the class Zab1_0Test method deserializeSnapshot.

private void deserializeSnapshot(InputArchive ia) throws IOException {
    ZKDatabase zkdb = new ZKDatabase(null);
    zkdb.deserializeSnapshot(ia);
    String signature = ia.readString("signature");
    assertEquals("BenWasHere", signature);
}