Examples with BootstrapEventsIterator org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator used on opensource projects

Example 1 with BootstrapEventsIterator

use of org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator in project hive by apache.

the class ReplLoadTask method execute.

@Override
protected int execute(DriverContext driverContext) {
    try {
        int maxTasks = conf.getIntVar(HiveConf.ConfVars.REPL_APPROX_MAX_LOAD_TASKS);
        Context context = new Context(conf, getHive(), work.sessionStateLineageState, work.currentTransactionId);
        TaskTracker loadTaskTracker = new TaskTracker(maxTasks);
        /*
          for now for simplicity we are doing just one directory ( one database ), come back to use
          of multiple databases once we have the basic flow to chain creating of tasks in place for
          a database ( directory )
      */
        BootstrapEventsIterator iterator = work.iterator();
        ConstraintEventsIterator constraintIterator = work.constraintIterator();
        /*
      This is used to get hold of a reference during the current creation of tasks and is initialized
      with "0" tasks such that it will be non consequential in any operations done with task tracker
      compositions.
       */
        TaskTracker dbTracker = new TaskTracker(ZERO_TASKS);
        TaskTracker tableTracker = new TaskTracker(ZERO_TASKS);
        Scope scope = new Scope();
        boolean loadingConstraint = false;
        if (!iterator.hasNext() && constraintIterator.hasNext()) {
            loadingConstraint = true;
        }
        while ((iterator.hasNext() || (loadingConstraint && constraintIterator.hasNext())) && loadTaskTracker.canAddMoreTasks()) {
            BootstrapEvent next;
            if (!loadingConstraint) {
                next = iterator.next();
            } else {
                next = constraintIterator.next();
            }
            switch(next.eventType()) {
                case Database:
                    DatabaseEvent dbEvent = (DatabaseEvent) next;
                    dbTracker = new LoadDatabase(context, dbEvent, work.dbNameToLoadIn, loadTaskTracker).tasks();
                    loadTaskTracker.update(dbTracker);
                    if (work.hasDbState()) {
                        loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
                    }
                    work.updateDbEventState(dbEvent.toState());
                    scope.database = true;
                    scope.rootTasks.addAll(dbTracker.tasks());
                    dbTracker.debugLog("database");
                    break;
                case Table:
                    {
                        /*
              Implicit assumption here is that database level is processed first before table level,
              which will depend on the iterator used since it should provide the higher level directory
              listing before providing the lower level listing. This is also required such that
              the dbTracker /  tableTracker are setup correctly always.
           */
                        TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn, work.tableNameToLoadIn);
                        TableEvent tableEvent = (TableEvent) next;
                        LoadTable loadTable = new LoadTable(tableEvent, context, iterator.replLogger(), tableContext, loadTaskTracker);
                        tableTracker = loadTable.tasks();
                        if (!scope.database) {
                            scope.rootTasks.addAll(tableTracker.tasks());
                            scope.table = true;
                        }
                        setUpDependencies(dbTracker, tableTracker);
                        /*
            for table replication if we reach the max number of tasks then for the next run we will
            try to reload the same table again, this is mainly for ease of understanding the code
            as then we can avoid handling == > loading partitions for the table given that
            the creation of table lead to reaching max tasks vs,  loading next table since current
            one does not have partitions.
           */
                        // for a table we explicitly try to load partitions as there is no separate partitions events.
                        LoadPartitions loadPartitions = new LoadPartitions(context, iterator.replLogger(), loadTaskTracker, tableEvent, work.dbNameToLoadIn, tableContext);
                        TaskTracker partitionsTracker = loadPartitions.tasks();
                        partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, partitionsTracker);
                        tableTracker.debugLog("table");
                        partitionsTracker.debugLog("partitions for table");
                        break;
                    }
                case Partition:
                    {
                        /*
              This will happen only when loading tables and we reach the limit of number of tasks we can create;
              hence we know here that the table should exist and there should be a lastPartitionName
          */
                        PartitionEvent event = (PartitionEvent) next;
                        TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn, work.tableNameToLoadIn);
                        LoadPartitions loadPartitions = new LoadPartitions(context, iterator.replLogger(), tableContext, loadTaskTracker, event.asTableEvent(), work.dbNameToLoadIn, event.lastPartitionReplicated());
                        /*
               the tableTracker here should be a new instance and not an existing one as this can
               only happen when we break in between loading partitions.
           */
                        TaskTracker partitionsTracker = loadPartitions.tasks();
                        partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, partitionsTracker);
                        partitionsTracker.debugLog("partitions");
                        break;
                    }
                case Function:
                    {
                        LoadFunction loadFunction = new LoadFunction(context, iterator.replLogger(), (FunctionEvent) next, work.dbNameToLoadIn, dbTracker);
                        TaskTracker functionsTracker = loadFunction.tasks();
                        if (!scope.database) {
                            scope.rootTasks.addAll(functionsTracker.tasks());
                        } else {
                            setUpDependencies(dbTracker, functionsTracker);
                        }
                        loadTaskTracker.update(functionsTracker);
                        functionsTracker.debugLog("functions");
                        break;
                    }
                case Constraint:
                    {
                        LoadConstraint loadConstraint = new LoadConstraint(context, (ConstraintEvent) next, work.dbNameToLoadIn, dbTracker);
                        TaskTracker constraintTracker = loadConstraint.tasks();
                        scope.rootTasks.addAll(constraintTracker.tasks());
                        loadTaskTracker.update(constraintTracker);
                        constraintTracker.debugLog("constraints");
                    }
            }
            if (!loadingConstraint && !iterator.currentDbHasNext()) {
                createEndReplLogTask(context, scope, iterator.replLogger());
            }
        }
        boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState() || constraintIterator.hasNext();
        createBuilderTask(scope.rootTasks, addAnotherLoadTask);
        if (!iterator.hasNext() && !constraintIterator.hasNext()) {
            loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
            work.updateDbEventState(null);
        }
        this.childTasks = scope.rootTasks;
        /*
      Since there can be multiple rounds of this run all of which will be tied to the same
      query id -- generated in compile phase , adding a additional UUID to the end to print each run
      in separate files.
       */
        LOG.info("Root Tasks / Total Tasks : {} / {} ", childTasks.size(), loadTaskTracker.numberOfTasks());
        // Populate the driver context with the scratch dir info from the repl context, so that the temp dirs will be cleaned up later
        driverContext.getCtx().getFsScratchDirs().putAll(context.pathInfo.getFsScratchDirs());
    } catch (Exception e) {
        LOG.error("failed replication", e);
        setException(e);
        return 1;
    }
    LOG.info("completed load task run : {}", work.executedLoadTask());
    return 0;
}

Example 2 with BootstrapEventsIterator

use of org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator in project hive by apache.

the class ReplLoadTask method executeBootStrapLoad.

private int executeBootStrapLoad() throws Exception {
    int maxTasks = conf.getIntVar(HiveConf.ConfVars.REPL_APPROX_MAX_LOAD_TASKS);
    Context loadContext = new Context(work.dumpDirectory, conf, getHive(), work.sessionStateLineageState, context);
    TaskTracker loadTaskTracker = new TaskTracker(maxTasks);
    BootstrapEventsIterator iterator = work.bootstrapIterator();
    addLazyDataCopyTask(loadTaskTracker, iterator.replLogger());
    /*
        for now for simplicity we are doing just one directory ( one database ), come back to use
        of multiple databases once we have the basic flow to chain creating of tasks in place for
        a database ( directory )
    */
    ConstraintEventsIterator constraintIterator = work.constraintsIterator();
    /*
    This is used to get hold of a reference during the current creation of tasks and is initialized
    with "0" tasks such that it will be non consequential in any operations done with task tracker
    compositions.
     */
    TaskTracker dbTracker = new TaskTracker(ZERO_TASKS);
    TaskTracker tableTracker = new TaskTracker(ZERO_TASKS);
    Scope scope = new Scope();
    boolean loadingConstraint = false;
    if (!iterator.hasNext() && constraintIterator.hasNext()) {
        loadingConstraint = true;
    }
    boolean dbEventFound = false;
    while ((iterator.hasNext() || (loadingConstraint && constraintIterator.hasNext())) && loadTaskTracker.canAddMoreTasks()) {
        BootstrapEvent next;
        if (!loadingConstraint) {
            next = iterator.next();
        } else {
            next = constraintIterator.next();
        }
        switch(next.eventType()) {
            case Database:
                DatabaseEvent dbEvent = (DatabaseEvent) next;
                dbTracker = new LoadDatabase(loadContext, dbEvent, work.dbNameToLoadIn, loadTaskTracker, work.getMetricCollector()).tasks();
                loadTaskTracker.update(dbTracker);
                if (work.hasDbState()) {
                    loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, loadContext, scope));
                } else {
                    // Scope might have set to database in some previous iteration of loop, so reset it to false if database
                    // tracker has no tasks.
                    scope.database = false;
                }
                work.updateDbEventState(dbEvent.toState());
                if (dbTracker.hasTasks()) {
                    scope.rootTasks.addAll(dbTracker.tasks());
                    scope.database = true;
                    dbEventFound = true;
                }
                dbTracker.debugLog("database");
                break;
            case Table:
                /*
          Implicit assumption here is that database level is processed first before table level,
          which will depend on the iterator used since it should provide the higher level directory
          listing before providing the lower level listing. This is also required such that
          the dbTracker /  tableTracker are setup correctly always.
       */
                TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn);
                FSTableEvent tableEvent = (FSTableEvent) next;
                if (TableType.VIRTUAL_VIEW.name().equals(tableEvent.getMetaData().getTable().getTableType())) {
                    tableTracker = new TaskTracker(1);
                    tableTracker.addTask(createViewTask(tableEvent.getMetaData(), work.dbNameToLoadIn, conf, (new Path(work.dumpDirectory).getParent()).toString(), work.getMetricCollector()));
                } else {
                    LoadTable loadTable = new LoadTable(tableEvent, loadContext, iterator.replLogger(), tableContext, loadTaskTracker, work.getMetricCollector());
                    tableTracker = loadTable.tasks(work.isIncrementalLoad());
                }
                setUpDependencies(dbTracker, tableTracker);
                if (!scope.database && tableTracker.hasTasks()) {
                    scope.rootTasks.addAll(tableTracker.tasks());
                    scope.table = true;
                } else {
                    // Scope might have set to table in some previous iteration of loop, so reset it to false if table
                    // tracker has no tasks.
                    scope.table = false;
                }
                if (!TableType.VIRTUAL_VIEW.name().equals(tableEvent.getMetaData().getTable().getTableType())) {
                    /*
          for table replication if we reach the max number of tasks then for the next run we will
          try to reload the same table again, this is mainly for ease of understanding the code
          as then we can avoid handling == > loading partitions for the table given that
          the creation of table lead to reaching max tasks vs,  loading next table since current
          one does not have partitions.
         */
                    // for a table we explicitly try to load partitions as there is no separate partitions events.
                    LoadPartitions loadPartitions = new LoadPartitions(loadContext, iterator.replLogger(), loadTaskTracker, tableEvent, work.dbNameToLoadIn, tableContext, work.getMetricCollector());
                    TaskTracker partitionsTracker = loadPartitions.tasks();
                    partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, partitionsTracker);
                    tableTracker.debugLog("table");
                    partitionsTracker.debugLog("partitions for table");
                }
                break;
            case Partition:
                /*
          This will happen only when loading tables and we reach the limit of number of tasks we can create;
          hence we know here that the table should exist and there should be a lastPartitionName
      */
                addLoadPartitionTasks(loadContext, next, dbTracker, iterator, scope, loadTaskTracker, tableTracker);
                break;
            case Function:
                loadTaskTracker.update(addLoadFunctionTasks(loadContext, iterator, next, dbTracker, scope));
                break;
            case Constraint:
                loadTaskTracker.update(addLoadConstraintsTasks(loadContext, next, dbTracker, scope));
                break;
            default:
                break;
        }
        if (!loadingConstraint && !iterator.currentDbHasNext()) {
            createEndReplLogTask(loadContext, scope, iterator.replLogger());
        }
        if (dbEventFound && conf.getBoolVar(HiveConf.ConfVars.REPL_RETAIN_CUSTOM_LOCATIONS_FOR_DB_ON_TARGET)) {
            // Force the database creation before the other event like table/partition etc, so that data copy path creation
            // can be achieved.
            LOG.info("Database event found, will be processed exclusively");
            break;
        }
    }
    boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState() || constraintIterator.hasNext();
    if (addAnotherLoadTask) {
        createBuilderTask(scope.rootTasks);
    }
    // last repl ID of the database.
    if (!iterator.hasNext() && !constraintIterator.hasNext() && !work.isIncrementalLoad()) {
        loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, loadContext, scope));
        work.updateDbEventState(null);
    }
    if (childTasks == null) {
        childTasks = new ArrayList<>();
    }
    childTasks.addAll(scope.rootTasks);
    /*
    Since there can be multiple rounds of this run all of which will be tied to the same
    query id -- generated in compile phase , adding a additional UUID to the end to print each run
    in separate files.
     */
    LOG.info("Root Tasks / Total Tasks : {} / {} ", childTasks.size(), loadTaskTracker.numberOfTasks());
    // Populate the driver context with the scratch dir info from the repl context, so that the
    // temp dirs will be cleaned up later
    context.getFsScratchDirs().putAll(loadContext.pathInfo.getFsScratchDirs());
    if (!HiveConf.getBoolVar(conf, REPL_DUMP_SKIP_IMMUTABLE_DATA_COPY)) {
        createReplLoadCompleteAckTask();
    }
    LOG.info("completed load task run : {}", work.executedLoadTask());
    if (conf.getBoolVar(REPL_SNAPSHOT_DIFF_FOR_EXTERNAL_TABLE_COPY)) {
        Path snapPath = SnapshotUtils.getSnapshotFileListPath(new Path(work.dumpDirectory));
        try {
            SnapshotUtils.getDFS(getExternalTableBaseDir(conf), conf).rename(new Path(snapPath, EximUtil.FILE_LIST_EXTERNAL_SNAPSHOT_CURRENT), new Path(snapPath, EximUtil.FILE_LIST_EXTERNAL_SNAPSHOT_OLD), Options.Rename.OVERWRITE);
        } catch (FileNotFoundException fnf) {
        // Ignore if no file.
        }
    }
    return 0;
}