Examples with NestedScriptParser org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser used on opensource projects

Example 1 with NestedScriptParser

use of org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser in project hive by apache.

the class TestSchemaTool method testPostgresFilter.

/**
 * Test script formatting
 */
@Test
public void testPostgresFilter() throws Exception {
    String[] testScript = { "-- this is a comment", "DROP TABLE IF EXISTS fooTab;", HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT + ";", "CREATE TABLE fooTab(id INTEGER);", "DROP TABLE footab;", "-- ending comment" };
    String[] expectedScriptWithOptionPresent = { "DROP TABLE IF EXISTS fooTab", HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT, "CREATE TABLE fooTab(id INTEGER)", "DROP TABLE footab" };
    NestedScriptParser noDbOptParser = HiveSchemaHelper.getDbCommandParser("postgres", false);
    String expectedSQL = StringUtils.join(expectedScriptWithOptionPresent, System.getProperty("line.separator")) + System.getProperty("line.separator");
    File testScriptFile = generateTestScript(testScript);
    String flattenedSql = noDbOptParser.buildCommand(testScriptFile.getParentFile().getPath(), testScriptFile.getName());
    Assert.assertEquals(expectedSQL, flattenedSql);
    String[] expectedScriptWithOptionAbsent = { "DROP TABLE IF EXISTS fooTab", "CREATE TABLE fooTab(id INTEGER)", "DROP TABLE footab" };
    NestedScriptParser dbOptParser = HiveSchemaHelper.getDbCommandParser("postgres", PostgresCommandParser.POSTGRES_SKIP_STANDARD_STRINGS_DBOPT, null, null, null, null, false);
    expectedSQL = StringUtils.join(expectedScriptWithOptionAbsent, System.getProperty("line.separator")) + System.getProperty("line.separator");
    testScriptFile = generateTestScript(testScript);
    flattenedSql = dbOptParser.buildCommand(testScriptFile.getParentFile().getPath(), testScriptFile.getName());
    Assert.assertEquals(expectedSQL, flattenedSql);
}

Example 2 with NestedScriptParser

use of org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser in project hive by apache.

the class TestSchemaTool method testScriptWithDelimiter.

/**
 * Test script formatting
 */
@Test
public void testScriptWithDelimiter() throws Exception {
    String[] testScript = { "-- this is a comment", "DROP TABLE IF EXISTS fooTab;", "DELIMITER $$", "/*!1234 this is comment code like mysql */$$", "CREATE TABLE fooTab(id INTEGER)$$", "CREATE PROCEDURE fooProc()", "SELECT * FROM fooTab;", "CALL barProc();", "END PROCEDURE$$", "DELIMITER ;", "DROP TABLE footab;", "-- ending comment" };
    String[] resultScript = { "DROP TABLE IF EXISTS fooTab", "/*!1234 this is comment code like mysql */", "CREATE TABLE fooTab(id INTEGER)", "CREATE PROCEDURE fooProc()" + " " + "SELECT * FROM fooTab;" + " " + "CALL barProc();" + " " + "END PROCEDURE", "DROP TABLE footab" };
    String expectedSQL = StringUtils.join(resultScript, System.getProperty("line.separator")) + System.getProperty("line.separator");
    File testScriptFile = generateTestScript(testScript);
    NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql", false);
    String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(), testScriptFile.getName());
    Assert.assertEquals(expectedSQL, flattenedSql);
}

Example 3 with NestedScriptParser

use of org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser in project hive by apache.

the class TestSchemaTool method testScriptMultiRowComment.

/**
 * Test script formatting
 */
@Test
public void testScriptMultiRowComment() throws Exception {
    String[] testScript = { "-- this is a comment", "DROP TABLE IF EXISTS fooTab;", "DELIMITER $$", "/*!1234 this is comment code like mysql */$$", "CREATE TABLE fooTab(id INTEGER)$$", "DELIMITER ;", "/* multiline comment started ", " * multiline comment continue", " * multiline comment ended */", "DROP TABLE footab;", "-- ending comment" };
    String[] parsedScript = { "DROP TABLE IF EXISTS fooTab", "/*!1234 this is comment code like mysql */", "CREATE TABLE fooTab(id INTEGER)", "DROP TABLE footab" };
    String expectedSQL = StringUtils.join(parsedScript, System.getProperty("line.separator")) + System.getProperty("line.separator");
    File testScriptFile = generateTestScript(testScript);
    NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql", false);
    String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(), testScriptFile.getName());
    Assert.assertEquals(expectedSQL, flattenedSql);
}

Example 4 with NestedScriptParser

use of org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser in project hive by apache.

the class HiveSchemaTool method execSql.

/**
 * Run beeline with the given metastore script. Flatten the nested scripts
 * into single file.
 */
@Override
protected void execSql(String scriptDir, String scriptFile) throws IOException, HiveMetaException {
    NestedScriptParser dbCommandParser = getDbCommandParser(dbType, metaDbType);
    // expand the nested script
    // If the metaDbType is set, this is setting up the information
    // schema in Hive. That specifically means that the sql commands need
    // to be adjusted for the underlying RDBMS (correct quotation
    // strings, etc).
    String sqlCommands = dbCommandParser.buildCommand(scriptDir, scriptFile, metaDbType != null);
    File tmpFile = File.createTempFile("schematool", ".sql");
    tmpFile.deleteOnExit();
    // write out the buffer into a file. Add beeline commands for autocommit and close
    FileWriter fstream = new FileWriter(tmpFile.getPath());
    BufferedWriter out = new BufferedWriter(fstream);
    out.write("!autocommit on" + System.getProperty("line.separator"));
    out.write(sqlCommands);
    out.write("!closeall" + System.getProperty("line.separator"));
    out.close();
    execSql(tmpFile.getPath());
}

Example 5 with NestedScriptParser

use of org.apache.hadoop.hive.metastore.tools.schematool.HiveSchemaHelper.NestedScriptParser in project hive by apache.

the class SchemaToolTaskValidate method findCreateTable.

@VisibleForTesting
List<String> findCreateTable(String path, List<String> tableList) throws Exception {
    if (!(new File(path)).exists()) {
        throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table");
    }
    List<String> subs = new ArrayList<>();
    NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(schemaTool.getDbType(), false);
    Pattern regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*");
    try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
        String line = null;
        while ((line = reader.readLine()) != null) {
            if (sp.isNestedScript(line)) {
                String subScript = sp.getScriptName(line);
                LOG.debug("Schema subscript " + subScript + " found");
                subs.add(subScript);
                continue;
            }
            // suppress multi-spaces
            line = line.replaceAll("( )+", " ");
            line = line.replaceAll("\\(", " ");
            line = line.replaceAll("IF NOT EXISTS ", "");
            line = line.replaceAll("`", "");
            line = line.replaceAll("'", "");
            line = line.replaceAll("\"", "");
            Matcher matcher = regexp.matcher(line);
            if (matcher.find()) {
                String table = matcher.group(2);
                if (schemaTool.getDbType().equals("derby")) {
                    table = table.replaceAll("APP\\.", "");
                }
                tableList.add(table.toLowerCase());
                LOG.debug("Found table " + table + " in the schema");
            }
        }
    } catch (IOException ex) {
        throw new Exception(ex.getMessage());
    }
    return subs;
}