Examples with Counters org.apache.hadoop.mapreduce.Counters used on opensource projects

Example 76 with Counters

use of org.apache.hadoop.mapreduce.Counters in project phoenix by apache.

the class NonParameterizedIndexScrutinyToolIT method testScrutinyOnArrayTypes.

@Test
public void testScrutinyOnArrayTypes() throws Exception {
    String dataTableName = generateUniqueName();
    String indexTableName = generateUniqueName();
    String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, " + "VB VARBINARY)";
    String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (VB)";
    String upsertData = "UPSERT INTO %s VALUES (?, ?, ?)";
    String upsertIndex = "UPSERT INTO %s (\"0:NAME\", \":ID\", \"0:VB\") values (?,?,?)";
    try (Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
        conn.createStatement().execute(String.format(dataTableDDL, dataTableName));
        conn.createStatement().execute(String.format(indexTableDDL, indexTableName, dataTableName));
        // insert two rows
        PreparedStatement upsertDataStmt = conn.prepareStatement(String.format(upsertData, dataTableName));
        upsertRow(upsertDataStmt, 1, "name-1", new byte[] { 127, 0, 0, 1 });
        upsertRow(upsertDataStmt, 2, "name-2", new byte[] { 127, 1, 0, 5 });
        conn.commit();
        List<Job> completedJobs = runScrutiny(null, dataTableName, indexTableName);
        Job job = completedJobs.get(0);
        assertTrue(job.isSuccessful());
        Counters counters = job.getCounters();
        assertEquals(2, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT));
        // Now insert a different varbinary row
        upsertRow(upsertDataStmt, 3, "name-3", new byte[] { 1, 1, 1, 1 });
        conn.commit();
        PreparedStatement upsertIndexStmt = conn.prepareStatement(String.format(upsertIndex, indexTableName));
        upsertIndexStmt.setString(1, "name-3");
        upsertIndexStmt.setInt(2, 3);
        upsertIndexStmt.setBytes(3, new byte[] { 0, 0, 0, 1 });
        upsertIndexStmt.executeUpdate();
        conn.commit();
        completedJobs = runScrutiny(null, dataTableName, indexTableName);
        job = completedJobs.get(0);
        assertTrue(job.isSuccessful());
        counters = job.getCounters();
        assertEquals(2, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(1, getCounterValue(counters, INVALID_ROW_COUNT));
        // Have source null
        upsertRow(upsertDataStmt, 4, "name-4", null);
        conn.commit();
        upsertIndexStmt.setString(1, "name-4");
        upsertIndexStmt.setInt(2, 4);
        upsertIndexStmt.setBytes(3, new byte[] { 0, 0, 1, 1 });
        upsertIndexStmt.executeUpdate();
        conn.commit();
        completedJobs = runScrutiny(null, dataTableName, indexTableName);
        job = completedJobs.get(0);
        assertTrue(job.isSuccessful());
        counters = job.getCounters();
        assertEquals(2, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(2, getCounterValue(counters, INVALID_ROW_COUNT));
        // Have target null
        upsertRow(upsertDataStmt, 5, "name-5", new byte[] { 0, 1, 1, 1 });
        conn.commit();
        upsertIndexStmt.setString(1, "name-5");
        upsertIndexStmt.setInt(2, 5);
        upsertIndexStmt.setBytes(3, null);
        upsertIndexStmt.executeUpdate();
        conn.commit();
        completedJobs = runScrutiny(null, dataTableName, indexTableName);
        job = completedJobs.get(0);
        assertTrue(job.isSuccessful());
        counters = job.getCounters();
        assertEquals(2, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(3, getCounterValue(counters, INVALID_ROW_COUNT));
        // Have both of them null
        upsertRow(upsertDataStmt, 6, "name-6", null);
        conn.commit();
        upsertIndexStmt.setString(1, "name-6");
        upsertIndexStmt.setInt(2, 6);
        upsertIndexStmt.setBytes(3, null);
        upsertIndexStmt.executeUpdate();
        conn.commit();
        completedJobs = runScrutiny(null, dataTableName, indexTableName);
        job = completedJobs.get(0);
        assertTrue(job.isSuccessful());
        counters = job.getCounters();
        assertEquals(3, getCounterValue(counters, VALID_ROW_COUNT));
        assertEquals(3, getCounterValue(counters, INVALID_ROW_COUNT));
    }
}

Example 77 with Counters

use of org.apache.hadoop.mapreduce.Counters in project phoenix by apache.

the class LogicalTableNameIT method testUpdatePhysicalTableNameWithViews_runScrutiny.

@Test
public void testUpdatePhysicalTableNameWithViews_runScrutiny() throws Exception {
    try (Connection conn = getConnection(props)) {
        try (Connection conn2 = getConnection(props)) {
            String schemaName = "S_" + generateUniqueName();
            String tableName = "TBL_" + generateUniqueName();
            String view1Name = "VW1_" + generateUniqueName();
            String view1IndexName1 = "VW1IDX1_" + generateUniqueName();
            String view1IndexName2 = "VW1IDX2_" + generateUniqueName();
            String view2Name = "VW2_" + generateUniqueName();
            String view2IndexName1 = "VW2IDX1_" + generateUniqueName();
            testWithViewsAndIndex_BaseTableChange(conn, conn2, null, schemaName, tableName, view1Name, view1IndexName1, view1IndexName2, view2Name, view2IndexName1, false, createChildAfterRename);
            List<Job> completedJobs = IndexScrutinyToolBaseIT.runScrutinyTool(schemaName, view2Name, view2IndexName1, 1L, IndexScrutinyTool.SourceTable.DATA_TABLE_SOURCE);
            Job job = completedJobs.get(0);
            assertTrue(job.isSuccessful());
            Counters counters = job.getCounters();
            if (createChildAfterRename) {
                assertEquals(3, counters.findCounter(VALID_ROW_COUNT).getValue());
                assertEquals(2, counters.findCounter(INVALID_ROW_COUNT).getValue());
            } else {
                // Since we didn't build the index, we expect 1 missing index row and 2 are from the other index
                assertEquals(2, counters.findCounter(VALID_ROW_COUNT).getValue());
                assertEquals(3, counters.findCounter(INVALID_ROW_COUNT).getValue());
            }
        }
    }
}

Example 78 with Counters

use of org.apache.hadoop.mapreduce.Counters in project cdap by caskdata.

the class MapReduceMetricsWriter method reportStats.

public void reportStats() throws IOException, InterruptedException {
    Counters jobCounters = jobConf.getCounters();
    reportMapredStats(jobCounters);
}

Example 79 with Counters

use of org.apache.hadoop.mapreduce.Counters in project incubator-rya by apache.

the class IndexWritingTool method run.

@Override
public int run(final String[] args) throws Exception {
    Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix.");
    final String inputDir = PathUtils.clean(args[0]);
    final String sparqlFile = PathUtils.clean(args[1]);
    final String instStr = args[2];
    final String zooStr = args[3];
    final String userStr = args[4];
    final String passStr = args[5];
    final String tablePrefix = args[6];
    final String sparql = FileUtils.readFileToString(new File(sparqlFile));
    final Job job = new Job(getConf(), "Write HDFS Index to Accumulo");
    job.setJarByClass(this.getClass());
    final Configuration jobConf = job.getConfiguration();
    jobConf.setBoolean("mapred.map.tasks.speculative.execution", false);
    setVarOrders(sparql, jobConf);
    TextInputFormat.setInputPaths(job, inputDir);
    job.setInputFormatClass(TextInputFormat.class);
    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Mutation.class);
    job.setNumReduceTasks(0);
    String tableName;
    if (zooStr.equals("mock")) {
        tableName = tablePrefix;
    } else {
        tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase();
    }
    setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName);
    jobConf.set(sparql_key, sparql);
    final int complete = job.waitForCompletion(true) ? 0 : -1;
    if (complete == 0) {
        final String[] varOrders = jobConf.getStrings("varOrders");
        final String orders = Joiner.on("\u0000").join(varOrders);
        Instance inst;
        if (zooStr.equals("mock")) {
            inst = new MockInstance(instStr);
        } else {
            inst = new ZooKeeperInstance(instStr, zooStr);
        }
        final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8));
        final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1);
        final Counters counters = job.getCounters();
        final Counter c1 = counters.findCounter(cardCounter, cardCounter);
        final Mutation m = new Mutation("~SPARQL");
        final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8));
        m.put(new Text("" + c1.getValue()), new Text(orders), v);
        bw.addMutation(m);
        bw.close();
        return complete;
    } else {
        return complete;
    }
}

Example 80 with Counters

use of org.apache.hadoop.mapreduce.Counters in project java-bigtable-hbase by googleapis.

the class ImportHBaseSnapshotJob method run.

/**
 * Job runner
 *
 * @param args command line input
 * @return job success flag
 * @throws Exception terminal for application failure
 */
@Override
public int run(String[] args) throws Exception {
    Configuration conf = HBaseConfiguration.create(getConf());
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (setConfFromArgs(conf, otherArgs) != 0)
        return -1;
    LOG.info("Using Arguments: " + "\n" + "snapshotName = " + conf.get(SNAPSHOTNAME_KEY) + "\n" + "snapshotDir = " + conf.get(HConstants.HBASE_DIR) + "\n" + "bigtableName = " + conf.get(TableOutputFormat.OUTPUT_TABLE) + "\n" + "tempDir = " + conf.get(SNAPSHOT_RESTOREDIR_KEY) + "\n" + CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY + " = " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) + "\n");
    Job job = createSubmittableJob(conf);
    int successStatus = (job.waitForCompletion(true) ? 0 : 1);
    Counters counters = job.getCounters();
    long numRows = counters.findCounter(ScanCounter.NUM_ROWS).getValue();
    long numCells = counters.findCounter(ScanCounter.NUM_CELLS).getValue();
    LOG.info("############# Num of ROWS imported= " + numRows);
    LOG.info("############# Num of CELLS imported= " + numCells);
    return successStatus;
}