Examples with ZooKeeper org.apache.zookeeper_voltpatches.ZooKeeper used on opensource projects

Example 11 with ZooKeeper

use of org.apache.zookeeper_voltpatches.ZooKeeper in project voltdb by VoltDB.

the class TestLeaderCache method testModifyChild.

@Test
public void testModifyChild() throws Exception {
    ZooKeeper zk = getClient(0);
    configure("/cache03", zk);
    LeaderCache dut = new LeaderCache(zk, "/cache03");
    dut.start(true);
    Map<Integer, Long> cache = dut.pointInTimeCache();
    assertEquals("3 items cached.", 3, cache.size());
    assertEquals(12345678, dut.get(0).longValue());
    zk.setData("/cache03/0", Long.toString(23456789).getBytes(), -1);
    while (true) {
        if (dut.get(0) == 23456789) {
            break;
        }
    }
    assertEquals("3 items cached.", 3, cache.size());
    assertEquals(23456789L, dut.get(0).longValue());
    assertEquals(87654321L, dut.get(1).longValue());
    assertEquals(11223344L, dut.get(2).longValue());
    dut.shutdown();
    zk.close();
}

Example 12 with ZooKeeper

use of org.apache.zookeeper_voltpatches.ZooKeeper in project voltdb by VoltDB.

the class TestLeaderCache method testInitialCache.

@Test
public void testInitialCache() throws Exception {
    ZooKeeper zk = getClient(0);
    configure("/cache01", zk);
    LeaderCache dut = new LeaderCache(zk, "/cache01");
    dut.start(true);
    Map<Integer, Long> cache = dut.pointInTimeCache();
    assertEquals("3 items cached.", 3, cache.size());
    assertEquals(12345678L, dut.get(0).longValue());
    assertEquals(87654321L, dut.get(1).longValue());
    assertEquals(11223344L, dut.get(2).longValue());
    dut.shutdown();
    zk.close();
}

Example 13 with ZooKeeper

use of org.apache.zookeeper_voltpatches.ZooKeeper in project voltdb by VoltDB.

the class ZKUtil method getClient.

public static final ZooKeeper getClient(String zkAddress, int timeout, Set<Long> verbotenThreads) throws Exception {
    final Semaphore zkConnect = new Semaphore(0);
    ZooKeeper zk = new ZooKeeper(zkAddress, 2000, new Watcher() {

        @Override
        public void process(WatchedEvent event) {
            if (event.getState() == KeeperState.SyncConnected) {
                zkConnect.release();
            }
        }
    }, verbotenThreads);
    if (!zkConnect.tryAcquire(timeout, TimeUnit.MILLISECONDS)) {
        return null;
    }
    return zk;
}

Example 14 with ZooKeeper

use of org.apache.zookeeper_voltpatches.ZooKeeper in project voltdb by VoltDB.

the class RealVoltDB method readDeploymentAndCreateStarterCatalogContext.

boolean readDeploymentAndCreateStarterCatalogContext(VoltDB.Configuration config) {
    /*
         * Debate with the cluster what the deployment file should be
         */
    try {
        ZooKeeper zk = m_messenger.getZK();
        byte[] deploymentBytes = null;
        try {
            deploymentBytes = org.voltcore.utils.CoreUtils.urlToBytes(m_config.m_pathToDeployment);
        } catch (Exception ex) {
        //Let us get bytes from ZK
        }
        DeploymentType deployment = null;
        try {
            if (deploymentBytes != null) {
                CatalogUtil.writeCatalogToZK(zk, // Fill in innocuous values for non-deployment stuff
                0, 0L, 0L, // spin loop in Inits.LoadCatalog.run() needs
                new byte[] {}, // this to be of zero length until we have a real catalog.
                null, deploymentBytes);
                hostLog.info("URL of deployment: " + m_config.m_pathToDeployment);
            } else {
                CatalogAndIds catalogStuff = CatalogUtil.getCatalogFromZK(zk);
                deploymentBytes = catalogStuff.deploymentBytes;
            }
        } catch (KeeperException.NodeExistsException e) {
            CatalogAndIds catalogStuff = CatalogUtil.getCatalogFromZK(zk);
            byte[] deploymentBytesTemp = catalogStuff.deploymentBytes;
            if (deploymentBytesTemp != null) {
                //We will ignore the supplied or default deployment anyways.
                if (deploymentBytes != null && !m_config.m_deploymentDefault) {
                    byte[] deploymentHashHere = CatalogUtil.makeDeploymentHash(deploymentBytes);
                    if (!(Arrays.equals(deploymentHashHere, catalogStuff.getDeploymentHash()))) {
                        hostLog.warn("The locally provided deployment configuration did not " + " match the configuration information found in the cluster.");
                    } else {
                        hostLog.info("Deployment configuration pulled from other cluster node.");
                    }
                }
                //Use remote deployment obtained.
                deploymentBytes = deploymentBytesTemp;
            } else {
                hostLog.error("Deployment file could not be loaded locally or remotely, " + "local supplied path: " + m_config.m_pathToDeployment);
                deploymentBytes = null;
            }
        } catch (KeeperException.NoNodeException e) {
            // no deploymentBytes case is handled below. So just log this error.
            if (hostLog.isDebugEnabled()) {
                hostLog.debug("Error trying to get deployment bytes from cluster", e);
            }
        }
        if (deploymentBytes == null) {
            hostLog.error("Deployment information could not be obtained from cluster node or locally");
            VoltDB.crashLocalVoltDB("No such deployment file: " + m_config.m_pathToDeployment, false, null);
        }
        if (deployment == null) {
            deployment = CatalogUtil.getDeployment(new ByteArrayInputStream(deploymentBytes));
        }
        // wasn't a valid xml deployment file
        if (deployment == null) {
            hostLog.error("Not a valid XML deployment file at URL: " + m_config.m_pathToDeployment);
            VoltDB.crashLocalVoltDB("Not a valid XML deployment file at URL: " + m_config.m_pathToDeployment, false, null);
        }
        /*
             * Check for invalid deployment file settings (enterprise-only) in the community edition.
             * Trick here is to print out all applicable problems and then stop, rather than stopping
             * after the first one is found.
             */
        if (!m_config.m_isEnterprise) {
            boolean shutdownDeployment = false;
            boolean shutdownAction = false;
            // check license features for community version
            if ((deployment.getCluster() != null) && (deployment.getCluster().getKfactor() > 0)) {
                consoleLog.error("K-Safety is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getSnapshot() != null) && (deployment.getSnapshot().isEnabled())) {
                consoleLog.error("Snapshots are not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getCommandlog() != null) && (deployment.getCommandlog().isEnabled())) {
                consoleLog.error("Command logging is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getExport() != null) && deployment.getExport().getConfiguration() != null && !deployment.getExport().getConfiguration().isEmpty()) {
                consoleLog.error("Export is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            // check the start action for the community edition
            if (m_config.m_startAction != StartAction.CREATE) {
                consoleLog.error("Start action \"" + m_config.m_startAction.getClass().getSimpleName() + "\" is not supported in the community edition of VoltDB.");
                shutdownAction = true;
            }
            // if the process needs to stop, try to be helpful
            if (shutdownAction || shutdownDeployment) {
                String msg = "This process will exit. Please run VoltDB with ";
                if (shutdownDeployment) {
                    msg += "a deployment file compatible with the community edition";
                }
                if (shutdownDeployment && shutdownAction) {
                    msg += " and ";
                }
                if (shutdownAction && !shutdownDeployment) {
                    msg += "the CREATE start action";
                }
                msg += ".";
                VoltDB.crashLocalVoltDB(msg, false, null);
            }
        }
        // note the heart beats are specified in seconds in xml, but ms internally
        HeartbeatType hbt = deployment.getHeartbeat();
        if (hbt != null) {
            m_config.m_deadHostTimeoutMS = hbt.getTimeout() * 1000;
            m_messenger.setDeadHostTimeout(m_config.m_deadHostTimeoutMS);
        } else {
            hostLog.info("Dead host timeout set to " + m_config.m_deadHostTimeoutMS + " milliseconds");
        }
        PartitionDetectionType pt = deployment.getPartitionDetection();
        if (pt != null) {
            m_config.m_partitionDetectionEnabled = pt.isEnabled();
            m_messenger.setPartitionDetectionEnabled(m_config.m_partitionDetectionEnabled);
        }
        // get any consistency settings into config
        ConsistencyType consistencyType = deployment.getConsistency();
        if (consistencyType != null) {
            m_config.m_consistencyReadLevel = Consistency.ReadLevel.fromReadLevelType(consistencyType.getReadlevel());
        }
        final String elasticSetting = deployment.getCluster().getElastic().trim().toUpperCase();
        if (elasticSetting.equals("ENABLED")) {
            TheHashinator.setConfiguredHashinatorType(HashinatorType.ELASTIC);
        } else if (!elasticSetting.equals("DISABLED")) {
            VoltDB.crashLocalVoltDB("Error in deployment file,  elastic attribute of " + "cluster element must be " + "'enabled' or 'disabled' but was '" + elasticSetting + "'", false, null);
        } else {
            TheHashinator.setConfiguredHashinatorType(HashinatorType.LEGACY);
        }
        // log system setting information
        SystemSettingsType sysType = deployment.getSystemsettings();
        if (sysType != null) {
            if (sysType.getElastic() != null) {
                hostLog.info("Elastic duration set to " + sysType.getElastic().getDuration() + " milliseconds");
                hostLog.info("Elastic throughput set to " + sysType.getElastic().getThroughput() + " mb/s");
            }
            if (sysType.getTemptables() != null) {
                hostLog.info("Max temptable size set to " + sysType.getTemptables().getMaxsize() + " mb");
            }
            if (sysType.getSnapshot() != null) {
                hostLog.info("Snapshot priority set to " + sysType.getSnapshot().getPriority() + " [0 - 10]");
            }
            if (sysType.getQuery() != null) {
                if (sysType.getQuery().getTimeout() > 0) {
                    hostLog.info("Query timeout set to " + sysType.getQuery().getTimeout() + " milliseconds");
                    m_config.m_queryTimeout = sysType.getQuery().getTimeout();
                } else if (sysType.getQuery().getTimeout() == 0) {
                    hostLog.info("Query timeout set to unlimited");
                    m_config.m_queryTimeout = 0;
                }
            }
        }
        // log a warning on console log if security setting is turned off, like durability warning.
        SecurityType securityType = deployment.getSecurity();
        if (securityType == null || !securityType.isEnabled()) {
            consoleLog.warn(SECURITY_OFF_WARNING);
        }
        // create a dummy catalog to load deployment info into
        Catalog catalog = new Catalog();
        // Need these in the dummy catalog
        Cluster cluster = catalog.getClusters().add("cluster");
        cluster.getDatabases().add("database");
        String result = CatalogUtil.compileDeployment(catalog, deployment, true);
        if (result != null) {
            // Any other non-enterprise deployment errors will be caught and handled here
            // (such as <= 0 host count)
            VoltDB.crashLocalVoltDB(result);
        }
        m_catalogContext = new CatalogContext(//txnid
        TxnEgo.makeZero(MpInitiator.MP_INIT_PID).getTxnId(), //timestamp
        0, catalog, new DbSettings(m_clusterSettings, m_nodeSettings), new byte[] {}, null, deploymentBytes, 0, m_messenger);
        return ((deployment.getCommandlog() != null) && (deployment.getCommandlog().isEnabled()));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 15 with ZooKeeper

use of org.apache.zookeeper_voltpatches.ZooKeeper in project voltdb by VoltDB.

the class MeshProber method considerMeshPlea.

@Override
public JoinAcceptor.PleaDecision considerMeshPlea(ZooKeeper zk, int hostId, JSONObject jo) {
    checkArgument(zk != null, "zookeeper is null");
    checkArgument(jo != null, "json object is null");
    if (!HostCriteria.hasCriteria(jo)) {
        return new JoinAcceptor.PleaDecision(String.format("Joining node version %s is incompatible with this node verion %s", jo.optString(SocketJoiner.VERSION_STRING, "(unknown)"), m_versionChecker.getVersionString()), false, false);
    }
    HostCriteria hc = new HostCriteria(jo);
    Map<Integer, HostCriteria> hostCriteria = m_hostCriteria.get();
    // when the cluster is forming anew)
    if (!getNodeState().operational() && !hostCriteria.values().stream().anyMatch(c -> c.getNodeState().operational())) {
        List<String> incompatibilities = asHostCriteria().listIncompatibilities(hc);
        if (!incompatibilities.isEmpty()) {
            Joiner joiner = Joiner.on("\n    ").skipNulls();
            String error = "Incompatible joining criteria:\n    " + joiner.join(incompatibilities);
            return new JoinAcceptor.PleaDecision(error, false, false);
        }
        return new JoinAcceptor.PleaDecision(null, true, false);
    } else {
        StartAction operationalStartAction = hostCriteria.values().stream().filter(c -> c.getNodeState().operational()).map(c -> c.getStartAction()).findFirst().orElse(getStartAction());
        if (operationalStartAction == StartAction.PROBE && hc.getStartAction() != StartAction.PROBE) {
            String msg = "Invalid VoltDB command. Please use init and start to join this cluster";
            return new JoinAcceptor.PleaDecision(msg, false, false);
        }
    }
    // how many hosts are already in the mesh?
    Stat stat = new Stat();
    try {
        zk.getChildren(CoreZK.hosts, false, stat);
    } catch (InterruptedException e) {
        String msg = "Interrupted while considering mesh plea";
        m_networkLog.error(msg, e);
        return new JoinAcceptor.PleaDecision(msg, false, false);
    } catch (KeeperException e) {
        EnumSet<KeeperException.Code> closing = EnumSet.of(KeeperException.Code.SESSIONEXPIRED, KeeperException.Code.CONNECTIONLOSS);
        if (closing.contains(e.code())) {
            return new JoinAcceptor.PleaDecision("Shutting down", false, false);
        } else {
            String msg = "Failed to list hosts while considering a mesh plea";
            m_networkLog.error(msg, e);
            return new JoinAcceptor.PleaDecision(msg, false, false);
        }
    }
    // connecting to already wholly formed cluster
    if (stat.getNumChildren() >= getHostCount()) {
        return new JoinAcceptor.PleaDecision(hc.isAddAllowed() ? null : "Cluster is already complete", hc.isAddAllowed(), false);
    } else if (stat.getNumChildren() < getHostCount()) {
        // check for concurrent rejoins
        final int rejoiningHost = CoreZK.createRejoinNodeIndicator(zk, hostId);
        if (rejoiningHost == -1) {
            return new JoinAcceptor.PleaDecision(null, true, false);
        } else {
            String msg = "Only one host can rejoin at a time. Host " + rejoiningHost + " is still rejoining.";
            return new JoinAcceptor.PleaDecision(msg, false, true);
        }
    }
    return new JoinAcceptor.PleaDecision(null, true, false);
}