Examples with Tasks org.apache.iceberg.util.Tasks used on opensource projects

Example 1 with Tasks

use of org.apache.iceberg.util.Tasks in project hive by apache.

the class HiveIcebergOutputCommitter method commitTable.

/**
 * Collects the additions to a single table and adds/commits the new files to the Iceberg table.
 * @param io The io to read the forCommit files
 * @param executor The executor used to read the forCommit files
 * @param jobContext The job context
 * @param name The name of the table used for loading from the catalog
 * @param location The location of the table used for loading from the catalog
 * @param catalogName The name of the catalog that contains the table
 */
private void commitTable(FileIO io, ExecutorService executor, JobContext jobContext, String name, String location, String catalogName) {
    JobConf conf = jobContext.getJobConf();
    Properties catalogProperties = new Properties();
    catalogProperties.put(Catalogs.NAME, name);
    catalogProperties.put(Catalogs.LOCATION, location);
    if (catalogName != null) {
        catalogProperties.put(InputFormatConfig.CATALOG_NAME, catalogName);
    }
    Table table = Catalogs.loadTable(conf, catalogProperties);
    long startTime = System.currentTimeMillis();
    LOG.info("Committing job has started for table: {}, using location: {}", table, generateJobLocation(location, conf, jobContext.getJobID()));
    int numTasks = SessionStateUtil.getCommitInfo(conf, name).map(info -> info.getTaskNum()).orElseGet(() -> {
        // Fallback logic, if number of tasks are not available in the config
        // If there are reducers, then every reducer will generate a result file.
        // If this is a map only task, then every mapper will generate a result file.
        LOG.info("Number of tasks not available in session state for jobID: {}, table: {}. Falling back to jobConf " + "numReduceTasks/numMapTasks", jobContext.getJobID(), name);
        return conf.getNumReduceTasks() > 0 ? conf.getNumReduceTasks() : conf.getNumMapTasks();
    });
    Collection<DataFile> dataFiles = dataFiles(numTasks, executor, location, jobContext, io, true);
    boolean isOverwrite = conf.getBoolean(InputFormatConfig.IS_OVERWRITE, false);
    if (isOverwrite) {
        if (!dataFiles.isEmpty()) {
            ReplacePartitions overwrite = table.newReplacePartitions();
            dataFiles.forEach(overwrite::addFile);
            overwrite.commit();
            LOG.info("Overwrite commit took {} ms for table: {} with {} file(s)", System.currentTimeMillis() - startTime, table, dataFiles.size());
        } else if (table.spec().isUnpartitioned()) {
            // TODO: we won't get here if we have a formerly-partitioned table, whose partition specs have been turned void
            table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()).commit();
            LOG.info("Cleared table contents as part of empty overwrite for unpartitioned table. " + "Commit took {} ms for table: {}", System.currentTimeMillis() - startTime, table);
        }
        LOG.debug("Overwrote partitions with files {}", dataFiles);
    } else if (dataFiles.size() > 0) {
        // Appending data files to the table
        // We only create a new commit if there's something to append
        AppendFiles append = table.newAppend();
        dataFiles.forEach(append::appendFile);
        append.commit();
        LOG.info("Append commit took {} ms for table: {} with {} file(s)", System.currentTimeMillis() - startTime, table, dataFiles.size());
        LOG.debug("Added files {}", dataFiles);
    } else {
        LOG.info("Not creating a new commit for table: {}, jobID: {}, since there were no new files to append", table, jobContext.getJobID());
    }
}

Example 2 with Tasks

use of org.apache.iceberg.util.Tasks in project hive by apache.

the class HiveIcebergOutputCommitter method listForCommits.

/**
 * Lists the forCommit files under a job location. This should only be used by {@link #abortJob(JobContext, int)},
 * since on the Tez AM-side it will have no access to the correct number of writer tasks otherwise. The commitJob
 * should not need to use this listing as it should have access to the vertex status info on the HS2-side.
 * @param jobConf jobConf used for getting the FS
 * @param jobLocation The job location that we should list
 * @return The set of forCommit files under the job location
 * @throws IOException if the listing fails
 */
private Set<FileStatus> listForCommits(JobConf jobConf, String jobLocation) throws IOException {
    Path path = new Path(jobLocation);
    LOG.debug("Listing job location to get forCommits for abort: {}", jobLocation);
    FileStatus[] children = path.getFileSystem(jobConf).listStatus(path);
    LOG.debug("Listing the job location: {} yielded these files: {}", jobLocation, Arrays.toString(children));
    return Arrays.stream(children).filter(child -> !child.isDirectory() && child.getPath().getName().endsWith(FOR_COMMIT_EXTENSION)).collect(Collectors.toSet());
}