Examples with Cluster org.voltdb.catalog.Cluster used on opensource projects

Example 6 with Cluster

use of org.voltdb.catalog.Cluster in project voltdb by VoltDB.

the class RealVoltDB method readDeploymentAndCreateStarterCatalogContext.

boolean readDeploymentAndCreateStarterCatalogContext(VoltDB.Configuration config) {
    /*
         * Debate with the cluster what the deployment file should be
         */
    try {
        ZooKeeper zk = m_messenger.getZK();
        byte[] deploymentBytes = null;
        try {
            deploymentBytes = org.voltcore.utils.CoreUtils.urlToBytes(m_config.m_pathToDeployment);
        } catch (Exception ex) {
        //Let us get bytes from ZK
        }
        DeploymentType deployment = null;
        try {
            if (deploymentBytes != null) {
                CatalogUtil.writeCatalogToZK(zk, // Fill in innocuous values for non-deployment stuff
                0, 0L, 0L, // spin loop in Inits.LoadCatalog.run() needs
                new byte[] {}, // this to be of zero length until we have a real catalog.
                null, deploymentBytes);
                hostLog.info("URL of deployment: " + m_config.m_pathToDeployment);
            } else {
                CatalogAndIds catalogStuff = CatalogUtil.getCatalogFromZK(zk);
                deploymentBytes = catalogStuff.deploymentBytes;
            }
        } catch (KeeperException.NodeExistsException e) {
            CatalogAndIds catalogStuff = CatalogUtil.getCatalogFromZK(zk);
            byte[] deploymentBytesTemp = catalogStuff.deploymentBytes;
            if (deploymentBytesTemp != null) {
                //We will ignore the supplied or default deployment anyways.
                if (deploymentBytes != null && !m_config.m_deploymentDefault) {
                    byte[] deploymentHashHere = CatalogUtil.makeDeploymentHash(deploymentBytes);
                    if (!(Arrays.equals(deploymentHashHere, catalogStuff.getDeploymentHash()))) {
                        hostLog.warn("The locally provided deployment configuration did not " + " match the configuration information found in the cluster.");
                    } else {
                        hostLog.info("Deployment configuration pulled from other cluster node.");
                    }
                }
                //Use remote deployment obtained.
                deploymentBytes = deploymentBytesTemp;
            } else {
                hostLog.error("Deployment file could not be loaded locally or remotely, " + "local supplied path: " + m_config.m_pathToDeployment);
                deploymentBytes = null;
            }
        } catch (KeeperException.NoNodeException e) {
            // no deploymentBytes case is handled below. So just log this error.
            if (hostLog.isDebugEnabled()) {
                hostLog.debug("Error trying to get deployment bytes from cluster", e);
            }
        }
        if (deploymentBytes == null) {
            hostLog.error("Deployment information could not be obtained from cluster node or locally");
            VoltDB.crashLocalVoltDB("No such deployment file: " + m_config.m_pathToDeployment, false, null);
        }
        if (deployment == null) {
            deployment = CatalogUtil.getDeployment(new ByteArrayInputStream(deploymentBytes));
        }
        // wasn't a valid xml deployment file
        if (deployment == null) {
            hostLog.error("Not a valid XML deployment file at URL: " + m_config.m_pathToDeployment);
            VoltDB.crashLocalVoltDB("Not a valid XML deployment file at URL: " + m_config.m_pathToDeployment, false, null);
        }
        /*
             * Check for invalid deployment file settings (enterprise-only) in the community edition.
             * Trick here is to print out all applicable problems and then stop, rather than stopping
             * after the first one is found.
             */
        if (!m_config.m_isEnterprise) {
            boolean shutdownDeployment = false;
            boolean shutdownAction = false;
            // check license features for community version
            if ((deployment.getCluster() != null) && (deployment.getCluster().getKfactor() > 0)) {
                consoleLog.error("K-Safety is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getSnapshot() != null) && (deployment.getSnapshot().isEnabled())) {
                consoleLog.error("Snapshots are not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getCommandlog() != null) && (deployment.getCommandlog().isEnabled())) {
                consoleLog.error("Command logging is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            if ((deployment.getExport() != null) && deployment.getExport().getConfiguration() != null && !deployment.getExport().getConfiguration().isEmpty()) {
                consoleLog.error("Export is not supported " + "in the community edition of VoltDB.");
                shutdownDeployment = true;
            }
            // check the start action for the community edition
            if (m_config.m_startAction != StartAction.CREATE) {
                consoleLog.error("Start action \"" + m_config.m_startAction.getClass().getSimpleName() + "\" is not supported in the community edition of VoltDB.");
                shutdownAction = true;
            }
            // if the process needs to stop, try to be helpful
            if (shutdownAction || shutdownDeployment) {
                String msg = "This process will exit. Please run VoltDB with ";
                if (shutdownDeployment) {
                    msg += "a deployment file compatible with the community edition";
                }
                if (shutdownDeployment && shutdownAction) {
                    msg += " and ";
                }
                if (shutdownAction && !shutdownDeployment) {
                    msg += "the CREATE start action";
                }
                msg += ".";
                VoltDB.crashLocalVoltDB(msg, false, null);
            }
        }
        // note the heart beats are specified in seconds in xml, but ms internally
        HeartbeatType hbt = deployment.getHeartbeat();
        if (hbt != null) {
            m_config.m_deadHostTimeoutMS = hbt.getTimeout() * 1000;
            m_messenger.setDeadHostTimeout(m_config.m_deadHostTimeoutMS);
        } else {
            hostLog.info("Dead host timeout set to " + m_config.m_deadHostTimeoutMS + " milliseconds");
        }
        PartitionDetectionType pt = deployment.getPartitionDetection();
        if (pt != null) {
            m_config.m_partitionDetectionEnabled = pt.isEnabled();
            m_messenger.setPartitionDetectionEnabled(m_config.m_partitionDetectionEnabled);
        }
        // get any consistency settings into config
        ConsistencyType consistencyType = deployment.getConsistency();
        if (consistencyType != null) {
            m_config.m_consistencyReadLevel = Consistency.ReadLevel.fromReadLevelType(consistencyType.getReadlevel());
        }
        final String elasticSetting = deployment.getCluster().getElastic().trim().toUpperCase();
        if (elasticSetting.equals("ENABLED")) {
            TheHashinator.setConfiguredHashinatorType(HashinatorType.ELASTIC);
        } else if (!elasticSetting.equals("DISABLED")) {
            VoltDB.crashLocalVoltDB("Error in deployment file,  elastic attribute of " + "cluster element must be " + "'enabled' or 'disabled' but was '" + elasticSetting + "'", false, null);
        } else {
            TheHashinator.setConfiguredHashinatorType(HashinatorType.LEGACY);
        }
        // log system setting information
        SystemSettingsType sysType = deployment.getSystemsettings();
        if (sysType != null) {
            if (sysType.getElastic() != null) {
                hostLog.info("Elastic duration set to " + sysType.getElastic().getDuration() + " milliseconds");
                hostLog.info("Elastic throughput set to " + sysType.getElastic().getThroughput() + " mb/s");
            }
            if (sysType.getTemptables() != null) {
                hostLog.info("Max temptable size set to " + sysType.getTemptables().getMaxsize() + " mb");
            }
            if (sysType.getSnapshot() != null) {
                hostLog.info("Snapshot priority set to " + sysType.getSnapshot().getPriority() + " [0 - 10]");
            }
            if (sysType.getQuery() != null) {
                if (sysType.getQuery().getTimeout() > 0) {
                    hostLog.info("Query timeout set to " + sysType.getQuery().getTimeout() + " milliseconds");
                    m_config.m_queryTimeout = sysType.getQuery().getTimeout();
                } else if (sysType.getQuery().getTimeout() == 0) {
                    hostLog.info("Query timeout set to unlimited");
                    m_config.m_queryTimeout = 0;
                }
            }
        }
        // log a warning on console log if security setting is turned off, like durability warning.
        SecurityType securityType = deployment.getSecurity();
        if (securityType == null || !securityType.isEnabled()) {
            consoleLog.warn(SECURITY_OFF_WARNING);
        }
        // create a dummy catalog to load deployment info into
        Catalog catalog = new Catalog();
        // Need these in the dummy catalog
        Cluster cluster = catalog.getClusters().add("cluster");
        cluster.getDatabases().add("database");
        String result = CatalogUtil.compileDeployment(catalog, deployment, true);
        if (result != null) {
            // Any other non-enterprise deployment errors will be caught and handled here
            // (such as <= 0 host count)
            VoltDB.crashLocalVoltDB(result);
        }
        m_catalogContext = new CatalogContext(//txnid
        TxnEgo.makeZero(MpInitiator.MP_INIT_PID).getTxnId(), //timestamp
        0, catalog, new DbSettings(m_clusterSettings, m_nodeSettings), new byte[] {}, null, deploymentBytes, 0, m_messenger);
        return ((deployment.getCommandlog() != null) && (deployment.getCommandlog().isEnabled()));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 7 with Cluster

use of org.voltdb.catalog.Cluster in project voltdb by VoltDB.

the class TestEELibraryLoader method testLoader.

@Test
public void testLoader() {
    VoltDB.Configuration configuration = new VoltDB.Configuration();
    configuration.m_noLoadLibVOLTDB = true;
    MockVoltDB mockvolt = new MockVoltDB();
    VoltDB.ignoreCrash = true;
    VoltDB.replaceVoltDBInstanceForTest(mockvolt);
    mockvolt.m_noLoadLib = true;
    assertFalse(EELibraryLoader.loadExecutionEngineLibrary(false));
    assertFalse(VoltDB.wasCrashCalled);
    boolean threw = false;
    try {
        assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
    } catch (AssertionError ae) {
        threw = true;
    }
    assertTrue(threw);
    assertTrue(VoltDB.wasCrashCalled);
    VoltDB.wasCrashCalled = false;
    VoltDB.initialize(configuration);
    assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
    assertFalse(VoltDB.wasCrashCalled);
    // Now test SUCCESS case
    configuration = new VoltDB.Configuration();
    VoltDBInterface mockitovolt = mock(VoltDBInterface.class);
    VoltDBInterface realvolt = new RealVoltDB();
    when(mockitovolt.getEELibraryVersionString()).thenReturn(realvolt.getEELibraryVersionString());
    CatalogContext catContext = mock(CatalogContext.class);
    Cluster cluster = mock(Cluster.class);
    when(catContext.getCluster()).thenReturn(cluster);
    when(mockitovolt.getCatalogContext()).thenReturn(catContext);
    VoltDB.replaceVoltDBInstanceForTest(mockitovolt);
    VoltDB.initialize(configuration);
    assertTrue(EELibraryLoader.loadExecutionEngineLibrary(true));
}

Example 8 with Cluster

use of org.voltdb.catalog.Cluster in project voltdb by VoltDB.

the class VoltCompiler method compileInternal.

/**
     * Internal method for compiling with and without a project.xml file or DDL files.
     *
     * @param projectReader Reader for project file or null if a project file is not used.
     * @param ddlFilePaths The list of DDL files to compile (when no project is provided).
     * @param jarOutputRet The in-memory jar to populate or null if the caller doesn't provide one.
     * @return The InMemoryJarfile containing the compiled catalog if
     * successful, null if not.  If the caller provided an InMemoryJarfile, the
     * return value will be the same object, not a copy.
     */
private InMemoryJarfile compileInternal(final VoltCompilerReader cannonicalDDLIfAny, final Catalog previousCatalogIfAny, final List<VoltCompilerReader> ddlReaderList, final InMemoryJarfile jarOutputRet) {
    // Expect to have either >1 ddl file or a project file.
    assert (ddlReaderList.size() > 0);
    // Make a temporary local output jar if one wasn't provided.
    final InMemoryJarfile jarOutput = (jarOutputRet != null ? jarOutputRet : new InMemoryJarfile());
    if (ddlReaderList == null || ddlReaderList.isEmpty()) {
        addErr("One or more DDL files are required.");
        return null;
    }
    // clear out the warnings and errors
    m_warnings.clear();
    m_infos.clear();
    m_errors.clear();
    // do all the work to get the catalog
    final Catalog catalog = compileCatalogInternal(cannonicalDDLIfAny, previousCatalogIfAny, ddlReaderList, jarOutput);
    if (catalog == null) {
        return null;
    }
    Cluster cluster = catalog.getClusters().get("cluster");
    assert (cluster != null);
    Database database = cluster.getDatabases().get("database");
    assert (database != null);
    // Build DDL from Catalog Data
    String ddlWithBatchSupport = CatalogSchemaTools.toSchema(catalog, m_importLines);
    m_canonicalDDL = CatalogSchemaTools.toSchemaWithoutInlineBatches(ddlWithBatchSupport);
    // generate the catalog report and write it to disk
    try {
        generateCatalogReport(ddlWithBatchSupport);
    } catch (IOException e) {
        e.printStackTrace();
        return null;
    }
    jarOutput.put(AUTOGEN_DDL_FILE_NAME, m_canonicalDDL.getBytes(Constants.UTF8ENCODING));
    if (DEBUG_VERIFY_CATALOG) {
        debugVerifyCatalog(jarOutput, catalog);
    }
    // WRITE CATALOG TO JAR HERE
    final String catalogCommands = catalog.serialize();
    byte[] catalogBytes = catalogCommands.getBytes(Constants.UTF8ENCODING);
    try {
        // Note when upgrading the version has already been updated by the caller.
        if (!jarOutput.containsKey(CatalogUtil.CATALOG_BUILDINFO_FILENAME)) {
            addBuildInfo(jarOutput);
        }
        jarOutput.put(CatalogUtil.CATALOG_FILENAME, catalogBytes);
        // put the compiler report into the jarfile
        jarOutput.put("catalog-report.html", m_report.getBytes(Constants.UTF8ENCODING));
    } catch (final Exception e) {
        e.printStackTrace();
        return null;
    }
    assert (!hasErrors());
    if (hasErrors()) {
        return null;
    }
    return jarOutput;
}

Example 9 with Cluster

use of org.voltdb.catalog.Cluster in project voltdb by VoltDB.

the class CatalogUtil method setClusterInfo.

/**
     * Set cluster info in the catalog.
     * @param leader The leader hostname
     * @param catalog The catalog to be updated.
     * @param printLog Whether or not to print cluster configuration.
     */
private static void setClusterInfo(Catalog catalog, DeploymentType deployment) {
    ClusterType cluster = deployment.getCluster();
    int kFactor = cluster.getKfactor();
    Cluster catCluster = catalog.getClusters().get("cluster");
    // copy the deployment info that is currently not recorded anywhere else
    Deployment catDeploy = catCluster.getDeployment().get("deployment");
    catDeploy.setKfactor(kFactor);
    if (deployment.getPartitionDetection().isEnabled()) {
        catCluster.setNetworkpartition(true);
    } else {
        catCluster.setNetworkpartition(false);
    }
    setSystemSettings(deployment, catDeploy);
    catCluster.setHeartbeattimeout(deployment.getHeartbeat().getTimeout());
    // copy schema modification behavior from xml to catalog
    if (cluster.getSchema() != null) {
        catCluster.setUseddlschema(cluster.getSchema() == SchemaType.DDL);
    } else {
        // Don't think we can get here, deployment schema guarantees a default value
        hostLog.warn("Schema modification setting not found. " + "Forcing default behavior of UpdateCatalog to modify database schema.");
        catCluster.setUseddlschema(false);
    }
}

Example 10 with Cluster

use of org.voltdb.catalog.Cluster in project voltdb by VoltDB.

the class TestCatalogUtil method testSecurityEnabledFlag.

public void testSecurityEnabledFlag() throws Exception {
    final String secOff = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<deployment>" + "   <cluster hostcount='3' kfactor='1' sitesperhost='2'/>" + "   <paths><voltdbroot path=\"/tmp/" + System.getProperty("user.name") + "\" /></paths>" + "   <security enabled=\"false\"/>" + "</deployment>";
    final String secOnWithNoAdmin = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<deployment>" + "   <cluster hostcount='3' kfactor='1' sitesperhost='2'/>" + "   <paths><voltdbroot path=\"/tmp/" + System.getProperty("user.name") + "\" /></paths>" + "   <security enabled=\"true\"/>" + "   <users>" + "      <user name=\"joe\" password=\"aaa\"/>" + "   </users>" + "</deployment>";
    final String secOn = "<?xml version='1.0' encoding='UTF-8' standalone='no'?>" + "<deployment>" + "   <cluster hostcount='3' kfactor='1' sitesperhost='2'/>" + "   <paths><voltdbroot path=\"/tmp/" + System.getProperty("user.name") + "\" /></paths>" + "   <security enabled=\"true\"/>" + "   <users>" + "      <user name=\"joe\" password=\"aaa\" roles=\"administrator\"/>" + "   </users>" + "</deployment>";
    final File tmpSecOff = VoltProjectBuilder.writeStringToTempFile(secOff);
    CatalogUtil.compileDeployment(catalog, tmpSecOff.getPath(), false);
    Cluster cluster = catalog.getClusters().get("cluster");
    assertFalse(cluster.getSecurityenabled());
    setUp();
    final File tmpSecOnWithNoAdmin = VoltProjectBuilder.writeStringToTempFile(secOnWithNoAdmin);
    String result = CatalogUtil.compileDeployment(catalog, tmpSecOnWithNoAdmin.getPath(), false);
    assertTrue(result != null);
    assertTrue(result.contains("Cannot enable security without defining"));
    setUp();
    final File tmpSecOn = VoltProjectBuilder.writeStringToTempFile(secOn);
    CatalogUtil.compileDeployment(catalog, tmpSecOn.getPath(), false);
    cluster = catalog.getClusters().get("cluster");
    assertTrue(cluster.getSecurityenabled());
}