Examples with CatalogContext org.voltdb.CatalogContext used on opensource projects

Example 1 with CatalogContext

use of org.voltdb.CatalogContext in project voltdb by VoltDB.

the class UpdateCore method executePlanFragment.

@Override
public DependencyPair executePlanFragment(Map<Integer, List<VoltTable>> dependencies, long fragmentId, ParameterSet params, SystemProcedureExecutionContext context) {
    if (fragmentId == SysProcFragmentId.PF_updateCatalogPrecheckAndSync) {
        String[] tablesThatMustBeEmpty = (String[]) params.getParam(0);
        String[] reasonsForEmptyTables = (String[]) params.getParam(1);
        checkForNonEmptyTables(tablesThatMustBeEmpty, reasonsForEmptyTables, context);
        // Send out fragments to do the initial round-trip to synchronize
        // all the cluster sites on the start of catalog update, we'll do
        // the actual work on the *next* round-trip below
        // Don't actually care about the returned table, just need to send something
        // back to the MPI scoreboard
        DependencyPair success = new DependencyPair.TableDependencyPair(DEP_updateCatalogSync, new VoltTable(new ColumnInfo[] { new ColumnInfo("UNUSED", VoltType.BIGINT) }));
        if (!context.isLowestSiteId()) {
            // id on this host.
            if (log.isInfoEnabled()) {
                log.info("Site " + CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()) + " completed data precheck.");
            }
            return success;
        }
        // We know the ZK bytes are okay because the run() method wrote them before sending
        // out fragments
        CatalogAndIds catalogStuff = null;
        try {
            catalogStuff = CatalogUtil.getCatalogFromZK(VoltDB.instance().getHostMessenger().getZK());
            InMemoryJarfile testjar = new InMemoryJarfile(catalogStuff.catalogBytes);
            JarLoader testjarloader = testjar.getLoader();
            for (String classname : testjarloader.getClassNames()) {
                try {
                    m_javaClass.forName(classname, true, testjarloader);
                }// care about here.
                 catch (UnsupportedClassVersionError e) {
                    String msg = "Cannot load classes compiled with a higher version of Java than currently" + " in use. Class " + classname + " was compiled with ";
                    Integer major = 0;
                    try {
                        major = Integer.parseInt(e.getMessage().split("version")[1].trim().split("\\.")[0]);
                    } catch (Exception ex) {
                        log.debug("Unable to parse compile version number from UnsupportedClassVersionError.", ex);
                    }
                    if (m_versionMap.containsKey(major)) {
                        msg = msg.concat(m_versionMap.get(major) + ", current runtime version is " + System.getProperty("java.version") + ".");
                    } else {
                        msg = msg.concat("an incompatable Java version.");
                    }
                    log.error(msg);
                    throw new VoltAbortException(msg);
                } catch (LinkageError | ClassNotFoundException e) {
                    String cause = e.getMessage();
                    if (cause == null && e.getCause() != null) {
                        cause = e.getCause().getMessage();
                    }
                    String msg = "Error loading class: " + classname + " from catalog: " + e.getClass().getCanonicalName() + ", " + cause;
                    log.warn(msg);
                    throw new VoltAbortException(e);
                }
            }
        } catch (Exception e) {
            Throwables.propagate(e);
        }
        if (log.isInfoEnabled()) {
            log.info("Site " + CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()) + " completed data and catalog precheck.");
        }
        return success;
    } else if (fragmentId == SysProcFragmentId.PF_updateCatalogPrecheckAndSyncAggregate) {
        // Don't actually care about the returned table, just need to send something
        // back to the MPI scoreboard
        log.info("Site " + CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()) + " acknowledged data and catalog prechecks.");
        return new DependencyPair.TableDependencyPair(DEP_updateCatalogSyncAggregate, new VoltTable(new ColumnInfo[] { new ColumnInfo("UNUSED", VoltType.BIGINT) }));
    } else if (fragmentId == SysProcFragmentId.PF_updateCatalog) {
        String catalogDiffCommands = (String) params.toArray()[0];
        String commands = Encoder.decodeBase64AndDecompress(catalogDiffCommands);
        int expectedCatalogVersion = (Integer) params.toArray()[1];
        boolean requiresSnapshotIsolation = ((Byte) params.toArray()[2]) != 0;
        boolean requireCatalogDiffCmdsApplyToEE = ((Byte) params.toArray()[3]) != 0;
        boolean hasSchemaChange = ((Byte) params.toArray()[4]) != 0;
        boolean requiresNewExportGeneration = ((Byte) params.toArray()[5]) != 0;
        CatalogAndIds catalogStuff = null;
        try {
            catalogStuff = CatalogUtil.getCatalogFromZK(VoltDB.instance().getHostMessenger().getZK());
        } catch (Exception e) {
            Throwables.propagate(e);
        }
        String replayInfo = m_runner.getTxnState().isForReplay() ? " (FOR REPLAY)" : "";
        // if this is a new catalog, do the work to update
        if (context.getCatalogVersion() == expectedCatalogVersion) {
            // update the global catalog if we get there first
            @SuppressWarnings("deprecation") Pair<CatalogContext, CatalogSpecificPlanner> p = VoltDB.instance().catalogUpdate(commands, catalogStuff.catalogBytes, catalogStuff.getCatalogHash(), expectedCatalogVersion, DeprecatedProcedureAPIAccess.getVoltPrivateRealTransactionId(this), getUniqueId(), catalogStuff.deploymentBytes, catalogStuff.getDeploymentHash(), requireCatalogDiffCmdsApplyToEE, hasSchemaChange, requiresNewExportGeneration);
            // The producer would have been turned off by the code above already.
            if (VoltDB.instance().getReplicationRole() == ReplicationRole.NONE && !VoltDB.instance().getReplicationActive()) {
                context.resetDrAppliedTracker();
            }
            // update the local catalog.  Safe to do this thanks to the check to get into here.
            long uniqueId = m_runner.getUniqueId();
            long spHandle = m_runner.getTxnState().getNotice().getSpHandle();
            context.updateCatalog(commands, p.getFirst(), p.getSecond(), requiresSnapshotIsolation, uniqueId, spHandle, requireCatalogDiffCmdsApplyToEE, requiresNewExportGeneration);
            if (log.isDebugEnabled()) {
                log.debug(String.format("Site %s completed catalog update with catalog hash %s, deployment hash %s%s.", CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()), Encoder.hexEncode(catalogStuff.getCatalogHash()).substring(0, 10), Encoder.hexEncode(catalogStuff.getDeploymentHash()).substring(0, 10), replayInfo));
            }
        } else // if seen before by this code, then check to see if this is a restart
        if (context.getCatalogVersion() == (expectedCatalogVersion + 1) && Arrays.equals(context.getCatalogHash(), catalogStuff.getCatalogHash()) && Arrays.equals(context.getDeploymentHash(), catalogStuff.getDeploymentHash())) {
            log.info(String.format("Site %s will NOT apply an assumed restarted and identical catalog update with catalog hash %s and deployment hash %s.", CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()), Encoder.hexEncode(catalogStuff.getCatalogHash()), Encoder.hexEncode(catalogStuff.getDeploymentHash())));
        } else {
            VoltDB.crashLocalVoltDB("Invalid catalog update.  Expected version: " + expectedCatalogVersion + ", current version: " + context.getCatalogVersion(), false, null);
        }
        VoltTable result = new VoltTable(VoltSystemProcedure.STATUS_SCHEMA);
        result.addRow(VoltSystemProcedure.STATUS_OK);
        return new DependencyPair.TableDependencyPair(DEP_updateCatalog, result);
    } else if (fragmentId == SysProcFragmentId.PF_updateCatalogAggregate) {
        VoltTable result = VoltTableUtil.unionTables(dependencies.get(DEP_updateCatalog));
        return new DependencyPair.TableDependencyPair(DEP_updateCatalogAggregate, result);
    } else {
        VoltDB.crashLocalVoltDB("Received unrecognized plan fragment id " + fragmentId + " in UpdateApplicationCatalog", false, null);
    }
    throw new RuntimeException("Should not reach this code");
}

Example 2 with CatalogContext

use of org.voltdb.CatalogContext in project voltdb by VoltDB.

the class UpdateSettings method executePlanFragment.

@Override
public DependencyPair executePlanFragment(Map<Integer, List<VoltTable>> dependencies, long fragmentId, ParameterSet params, SystemProcedureExecutionContext context) {
    if (fragmentId == SysProcFragmentId.PF_updateSettingsBarrier) {
        DependencyPair success = new DependencyPair.TableDependencyPair(DEP_updateSettingsBarrier, new VoltTable(new ColumnInfo[] { new ColumnInfo("UNUSED", VoltType.BIGINT) }));
        if (log.isInfoEnabled()) {
            log.info("Site " + CoreUtils.hsIdToString(m_site.getCorrespondingSiteId()) + " reached settings update barrier.");
        }
        return success;
    } else if (fragmentId == SysProcFragmentId.PF_updateSettingsBarrierAggregate) {
        Object[] paramarr = params.toArray();
        byte[] settingsBytes = (byte[]) paramarr[0];
        int version = ((Integer) paramarr[1]).intValue();
        ZooKeeper zk = getHostMessenger().getZK();
        Stat stat = null;
        try {
            stat = zk.setData(VoltZK.cluster_settings, settingsBytes, version);
        } catch (KeeperException | InterruptedException e) {
            String msg = "Failed to update cluster settings";
            log.error(msg, e);
            throw new SettingsException(msg, e);
        }
        log.info("Saved new cluster settings state");
        return new DependencyPair.TableDependencyPair(DEP_updateSettingsBarrierAggregate, getVersionResponse(stat.getVersion()));
    } else if (fragmentId == SysProcFragmentId.PF_updateSettings) {
        Object[] paramarr = params.toArray();
        byte[] settingsBytes = (byte[]) paramarr[0];
        int version = ((Integer) paramarr[1]).intValue();
        ClusterSettings settings = ClusterSettings.create(settingsBytes);
        Pair<CatalogContext, CatalogSpecificPlanner> ctgdef = getVoltDB().settingsUpdate(settings, version);
        context.updateSettings(ctgdef.getFirst(), ctgdef.getSecond());
        VoltTable result = new VoltTable(VoltSystemProcedure.STATUS_SCHEMA);
        result.addRow(VoltSystemProcedure.STATUS_OK);
        return new DependencyPair.TableDependencyPair(DEP_updateSettings, result);
    } else if (fragmentId == SysProcFragmentId.PF_updateSettingsAggregate) {
        VoltTable result = VoltTableUtil.unionTables(dependencies.get(DEP_updateSettings));
        return new DependencyPair.TableDependencyPair(DEP_updateSettingsAggregate, result);
    } else {
        VoltDB.crashLocalVoltDB("Received unrecognized plan fragment id " + fragmentId + " in UpdateSettings", false, null);
    }
    throw new RuntimeException("Should not reach this code");
}

Example 3 with CatalogContext

use of org.voltdb.CatalogContext in project voltdb by VoltDB.

the class SwapTables method run.

public CompletableFuture<ClientResponse> run(String theTable, String otherTable) {
    String sql = "@SwapTables " + theTable + " " + otherTable;
    Object[] userParams = null;
    CatalogContext context = VoltDB.instance().getCatalogContext();
    List<String> sqlStatements = new ArrayList<>(1);
    sqlStatements.add(sql);
    return runNonDDLAdHoc(context, sqlStatements, true, null, ExplainMode.NONE, true, userParams);
}

Example 4 with CatalogContext

use of org.voltdb.CatalogContext in project voltdb by VoltDB.

the class VoltCompiler method upgradeCatalogAsNeeded.

/**
     * Check a loaded catalog. If it needs to be upgraded recompile it and save
     * an upgraded jar file.
     *
     * @param outputJar  in-memory jar file (updated in place here)
     * @return source version upgraded from or null if not upgraded
     * @throws IOException
     */
public String upgradeCatalogAsNeeded(InMemoryJarfile outputJar) throws IOException {
    // getBuildInfoFromJar() performs some validation.
    String[] buildInfoLines = CatalogUtil.getBuildInfoFromJar(outputJar);
    String versionFromCatalog = buildInfoLines[0];
    // Set if an upgrade happens.
    String upgradedFromVersion = null;
    // getConfig() may return null if it's being mocked for a test.
    if (VoltDB.Configuration.m_forceCatalogUpgrade || !versionFromCatalog.equals(VoltDB.instance().getVersionString())) {
        // Patch the buildinfo.
        String versionFromVoltDB = VoltDB.instance().getVersionString();
        buildInfoLines[0] = versionFromVoltDB;
        buildInfoLines[1] = String.format("voltdb-auto-upgrade-to-%s", versionFromVoltDB);
        byte[] buildInfoBytes = StringUtils.join(buildInfoLines, "\n").getBytes();
        outputJar.put(CatalogUtil.CATALOG_BUILDINFO_FILENAME, buildInfoBytes);
        // Gather DDL files for re-compilation
        List<VoltCompilerReader> ddlReaderList = new ArrayList<>();
        Entry<String, byte[]> entry = outputJar.firstEntry();
        while (entry != null) {
            String path = entry.getKey();
            // ddl files instead of using a brute force *.sql glob.
            if (path.toLowerCase().endsWith(".sql")) {
                ddlReaderList.add(new VoltCompilerJarFileReader(outputJar, path));
            }
            entry = outputJar.higherEntry(entry.getKey());
        }
        // Use the in-memory jarfile-provided class loader so that procedure
        // classes can be found and copied to the new file that gets written.
        ClassLoader originalClassLoader = m_classLoader;
        // Compile and save the file to voltdbroot. Assume it's a test environment if there
        // is no catalog context available.
        String jarName = String.format("catalog-%s.jar", versionFromVoltDB);
        String textName = String.format("catalog-%s.out", versionFromVoltDB);
        CatalogContext catalogContext = VoltDB.instance().getCatalogContext();
        final String outputJarPath = (catalogContext != null ? new File(VoltDB.instance().getVoltDBRootPath(), jarName).getPath() : VoltDB.Configuration.getPathToCatalogForTest(jarName));
        // Place the compiler output in a text file in the same folder.
        final String outputTextPath = (catalogContext != null ? new File(VoltDB.instance().getVoltDBRootPath(), textName).getPath() : VoltDB.Configuration.getPathToCatalogForTest(textName));
        try {
            m_classLoader = outputJar.getLoader();
            consoleLog.info(String.format("Version %s catalog will be automatically upgraded to version %s.", versionFromCatalog, versionFromVoltDB));
            // Do the compilation work.
            boolean success = compileInternalToFile(outputJarPath, null, null, ddlReaderList, outputJar);
            // Bomb out if we failed to generate the canonical DDL
            if (success) {
                boolean foundCanonicalDDL = false;
                entry = outputJar.firstEntry();
                while (entry != null) {
                    String path = entry.getKey();
                    if (path.toLowerCase().endsWith(".sql")) {
                        if (!path.toLowerCase().equals(AUTOGEN_DDL_FILE_NAME)) {
                            outputJar.remove(path);
                        } else {
                            foundCanonicalDDL = true;
                        }
                    }
                    entry = outputJar.higherEntry(entry.getKey());
                }
                success = foundCanonicalDDL;
            }
            if (success) {
                // Set up the return string.
                upgradedFromVersion = versionFromCatalog;
            }
            // Summarize the results to a file.
            // Briefly log success or failure and mention the output text file.
            PrintStream outputStream = new PrintStream(outputTextPath);
            try {
                if (success) {
                    summarizeSuccess(outputStream, outputStream, outputJarPath);
                    consoleLog.info(String.format("The catalog was automatically upgraded from " + "version %s to %s and saved to \"%s\". " + "Compiler output is available in \"%s\".", versionFromCatalog, versionFromVoltDB, outputJarPath, outputTextPath));
                } else {
                    summarizeErrors(outputStream, outputStream);
                    outputStream.close();
                    compilerLog.error("Catalog upgrade failed.");
                    compilerLog.info(String.format("Had attempted to perform an automatic version upgrade of a " + "catalog that was compiled by an older %s version of VoltDB, " + "but the automatic upgrade failed. The cluster  will not be " + "able to start until the incompatibility is fixed. " + "Try re-compiling the catalog with the newer %s version " + "of the VoltDB compiler. Compiler output from the failed " + "upgrade is available in \"%s\".", versionFromCatalog, versionFromVoltDB, outputTextPath));
                    throw new IOException(String.format("Catalog upgrade failed. You will need to recompile using voltdb compile."));
                }
            } finally {
                outputStream.close();
            }
        } catch (IOException ioe) {
            // Do nothing because this could come from the normal failure path
            throw ioe;
        } catch (Exception e) {
            compilerLog.error("Catalog upgrade failed with error:");
            compilerLog.error(e.getMessage());
            compilerLog.info(String.format("Had attempted to perform an automatic version upgrade of a " + "catalog that was compiled by an older %s version of VoltDB, " + "but the automatic upgrade failed. The cluster  will not be " + "able to start until the incompatibility is fixed. " + "Try re-compiling the catalog with the newer %s version " + "of the VoltDB compiler. Compiler output from the failed " + "upgrade is available in \"%s\".", versionFromCatalog, versionFromVoltDB, outputTextPath));
            throw new IOException(String.format("Catalog upgrade failed. You will need to recompile using voltdb compile."));
        } finally {
            // Restore the original class loader
            m_classLoader = originalClassLoader;
        }
    }
    return upgradedFromVersion;
}

Example 5 with CatalogContext

use of org.voltdb.CatalogContext in project voltdb by VoltDB.

the class VoltCompiler method generateCatalogReport.

private void generateCatalogReport(String ddlWithBatchSupport) throws IOException {
    VoltDBInterface voltdb = VoltDB.instance();
    // try to get a catalog context
    CatalogContext catalogContext = voltdb != null ? voltdb.getCatalogContext() : null;
    ClusterSettings clusterSettings = catalogContext != null ? catalogContext.getClusterSettings() : null;
    int tableCount = catalogContext != null ? catalogContext.tables.size() : 0;
    Deployment deployment = catalogContext != null ? catalogContext.cluster.getDeployment().get("deployment") : null;
    int hostcount = clusterSettings != null ? clusterSettings.hostcount() : 1;
    int kfactor = deployment != null ? deployment.getKfactor() : 0;
    int sitesPerHost = 8;
    if (voltdb != null && voltdb.getCatalogContext() != null) {
        sitesPerHost = voltdb.getCatalogContext().getNodeSettings().getLocalSitesCount();
    }
    boolean isPro = MiscUtils.isPro();
    long minHeapRqt = RealVoltDB.computeMinimumHeapRqt(isPro, tableCount, sitesPerHost, kfactor);
    m_report = ReportMaker.report(m_catalog, minHeapRqt, isPro, hostcount, sitesPerHost, kfactor, m_warnings, ddlWithBatchSupport);
    m_reportPath = null;
    File file = null;
    // write to working dir when using VoltCompiler directly
    if (standaloneCompiler) {
        file = new File("catalog-report.html");
    } else {
        // if we have a context, write report to voltroot
        if (catalogContext != null) {
            file = new File(VoltDB.instance().getVoltDBRootPath(), "catalog-report.html");
        }
    }
    // if there's a good place to write the report, do so
    if (file != null) {
        FileWriter fw = new FileWriter(file);
        fw.write(m_report);
        fw.close();
        m_reportPath = file.getAbsolutePath();
    }
}