Examples with WorkloadExecutor org.apache.phoenix.pherf.workload.WorkloadExecutor used on opensource projects

Example 1 with WorkloadExecutor

use of org.apache.phoenix.pherf.workload.WorkloadExecutor in project phoenix by apache.

the class DataIngestIT method testMultiTenantViewWriteWorkload.

@Test
public /**
     * Validates that Pherf can write data to a Multi-Tenant View in addition to 
     * standard Phoenix tables.
     */
void testMultiTenantViewWriteWorkload() throws Exception {
    // Arrange
    Scenario scenario = parser.getScenarioByName("testMTWriteScenario");
    WorkloadExecutor executor = new WorkloadExecutor();
    executor.add(new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO));
    // Act
    try {
        // Wait for data to load up.
        executor.get();
        executor.shutdown();
    } catch (Exception e) {
        fail("Failed to load data. An exception was thrown: " + e.getMessage());
    }
    assertExpectedNumberOfRecordsWritten(scenario);
}

Example 2 with WorkloadExecutor

use of org.apache.phoenix.pherf.workload.WorkloadExecutor in project phoenix by apache.

the class DataIngestIT method testRWWorkload.

@Test
public void testRWWorkload() throws Exception {
    Connection connection = util.getConnection();
    WorkloadExecutor executor = new WorkloadExecutor();
    DataModel dataModel = parser.getDataModelByName("test_scenario");
    List<DataModel> dataModels = new ArrayList<>();
    dataModels.add(dataModel);
    QueryExecutor qe = new QueryExecutor(parser, util, executor, dataModels, null, false);
    executor.add(qe);
    Scenario scenario = parser.getScenarioByName("testScenarioRW");
    String sql = "select count(*) from " + scenario.getTableName();
    try {
        // Wait for data to load up.
        executor.get();
        executor.shutdown();
        // Verify data has been loaded
        Integer count = new JdbcSession(connection).sql(sql).select(new Outcome<Integer>() {

            @Override
            public Integer handle(ResultSet resultSet, Statement statement) throws SQLException {
                while (resultSet.next()) {
                    return resultSet.getInt(1);
                }
                return null;
            }
        });
        assertNotNull("Could not retrieve count. " + count);
        // It would be better to sum up all the rowcounts for the scenarios, but this is fine
        assertTrue("Could not query any rows for in " + scenario.getTableName(), count > 0);
    } catch (Exception e) {
        fail("Failed to load data. An exception was thrown: " + e.getMessage());
    }
}

Example 3 with WorkloadExecutor

use of org.apache.phoenix.pherf.workload.WorkloadExecutor in project phoenix by apache.

the class DataIngestIT method testMultiTenantScenarioRunBeforeWriteWorkload.

@Test
public void testMultiTenantScenarioRunBeforeWriteWorkload() throws Exception {
    // Arrange
    Scenario scenario = parser.getScenarioByName("testMTDdlWriteScenario");
    WorkloadExecutor executor = new WorkloadExecutor();
    executor.add(new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO));
    // Act
    try {
        // Wait for data to load up.
        executor.get();
        executor.shutdown();
    } catch (Exception e) {
        fail("Failed to load data. An exception was thrown: " + e.getMessage());
    }
    assertExpectedNumberOfRecordsWritten(scenario);
}

Example 4 with WorkloadExecutor

use of org.apache.phoenix.pherf.workload.WorkloadExecutor in project phoenix by apache.

the class DataIngestIT method testColumnRulesApplied.

@Test
public void testColumnRulesApplied() {
    Scenario scenario = null;
    try {
        scenario = parser.getScenarioByName("testScenario");
        List<Column> columnListFromPhoenix = util.getColumnsFromPhoenix(scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), util.getConnection());
        assertTrue("Could not get phoenix columns.", columnListFromPhoenix.size() > 0);
        WriteWorkload loader = new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO);
        WorkloadExecutor executor = new WorkloadExecutor();
        executor.add(loader);
        executor.get();
        executor.shutdown();
        RulesApplier rulesApplier = loader.getRulesApplier();
        List<Map> modelList = rulesApplier.getModelList();
        assertTrue("Could not generate the modelList", modelList.size() > 0);
        for (Column column : columnListFromPhoenix) {
            DataValue data = rulesApplier.getDataForRule(scenario, column);
            // We are generating data values
            // so the value should have been specified by this point.
            assertTrue("Failed to retrieve data for column type: " + column.getType(), data != null);
            // so we should get the default rule.
            if ((column.getType() == DataTypeMapping.VARCHAR) && (column.getName().equals("NEWVAL_STRING"))) {
                assertTrue("Failed to retrieve data for column type: ", data.getDistribution() == Integer.MIN_VALUE);
            }
        }
        // Run some queries
        executor = new WorkloadExecutor();
        Workload query = new QueryExecutor(parser, util, executor);
        executor.add(query);
        executor.get();
        executor.shutdown();
        PhoenixUtil.create().deleteTables("ALL");
    } catch (Exception e) {
        fail("We had an exception: " + e.getMessage());
    }
}

Example 5 with WorkloadExecutor

use of org.apache.phoenix.pherf.workload.WorkloadExecutor in project phoenix by apache.

the class Pherf method run.

public void run() throws Exception {
    MonitorManager monitorManager = null;
    List<Workload> workloads = new ArrayList<>();
    WorkloadExecutor workloadExecutor = new WorkloadExecutor(properties, workloads, !isFunctional);
    try {
        if (listFiles) {
            ResourceList list = new ResourceList(PherfConstants.RESOURCE_DATAMODEL);
            Collection<Path> schemaFiles = list.getResourceList(PherfConstants.SCHEMA_ROOT_PATTERN + ".sql");
            System.out.println("Schema Files:");
            for (Path path : schemaFiles) {
                System.out.println(path);
            }
            list = new ResourceList(PherfConstants.RESOURCE_SCENARIO);
            Collection<Path> scenarioFiles = list.getResourceList(PherfConstants.SCENARIO_ROOT_PATTERN + ".xml");
            System.out.println("Scenario Files:");
            for (Path path : scenarioFiles) {
                System.out.println(path);
            }
            return;
        }
        // Compare results and exit  
        if (null != compareResults) {
            logger.info("\nStarting to compare results and exiting for " + compareResults);
            new GoogleChartGenerator(compareResults, compareType).readAndRender();
            return;
        }
        XMLConfigParser parser = new XMLConfigParser(scenarioFile);
        // Drop tables with PHERF schema and regex comparison
        if (null != dropPherfTablesRegEx) {
            logger.info("\nDropping existing table with PHERF namename and " + dropPherfTablesRegEx + " regex expression.");
            phoenixUtil.deleteTables(dropPherfTablesRegEx);
        }
        if (monitor) {
            monitorManager = new MonitorManager(Integer.parseInt(properties.getProperty("pherf.default.monitorFrequency")));
            workloadExecutor.add(monitorManager);
        }
        if (applySchema) {
            logger.info("\nStarting to apply schema...");
            SchemaReader reader = (schemaFile == null) ? new SchemaReader(".*.sql") : new SchemaReader(schemaFile);
            reader.applySchema();
        }
        // Schema and Data Load
        if (preLoadData) {
            logger.info("\nStarting Data Load...");
            Workload workload = new WriteWorkload(parser, generateStatistics);
            try {
                workloadExecutor.add(workload);
                // Wait for dataLoad to complete
                workloadExecutor.get(workload);
            } finally {
                if (null != workload) {
                    workload.complete();
                }
            }
        } else {
            logger.info("\nSKIPPED: Data Load and schema creation as -l argument not specified");
        }
        // Execute multi-threaded query sets
        if (executeQuerySets) {
            logger.info("\nStarting to apply Execute Queries...");
            workloadExecutor.add(new QueryExecutor(parser, phoenixUtil, workloadExecutor, parser.getDataModels(), queryHint, isFunctional, writeRuntimeResults));
        } else {
            logger.info("\nSKIPPED: Multithreaded query set execution as -q argument not specified");
        }
        // Clean up the monitor explicitly
        if (monitorManager != null) {
            logger.info("Run completed. Shutting down Monitor.");
            monitorManager.complete();
        }
        // Collect any final jobs
        workloadExecutor.get();
    } finally {
        if (workloadExecutor != null) {
            logger.info("Run completed. Shutting down thread pool.");
            workloadExecutor.shutdown();
        }
    }
}