Examples with LoadDatabase org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase used on opensource projects

Example 1 with LoadDatabase

use of org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase in project hive by apache.

the class ReplLoadTask method execute.

@Override
protected int execute(DriverContext driverContext) {
    try {
        int maxTasks = conf.getIntVar(HiveConf.ConfVars.REPL_APPROX_MAX_LOAD_TASKS);
        Context context = new Context(conf, getHive(), work.sessionStateLineageState, work.currentTransactionId);
        TaskTracker loadTaskTracker = new TaskTracker(maxTasks);
        /*
          for now for simplicity we are doing just one directory ( one database ), come back to use
          of multiple databases once we have the basic flow to chain creating of tasks in place for
          a database ( directory )
      */
        BootstrapEventsIterator iterator = work.iterator();
        ConstraintEventsIterator constraintIterator = work.constraintIterator();
        /*
      This is used to get hold of a reference during the current creation of tasks and is initialized
      with "0" tasks such that it will be non consequential in any operations done with task tracker
      compositions.
       */
        TaskTracker dbTracker = new TaskTracker(ZERO_TASKS);
        TaskTracker tableTracker = new TaskTracker(ZERO_TASKS);
        Scope scope = new Scope();
        boolean loadingConstraint = false;
        if (!iterator.hasNext() && constraintIterator.hasNext()) {
            loadingConstraint = true;
        }
        while ((iterator.hasNext() || (loadingConstraint && constraintIterator.hasNext())) && loadTaskTracker.canAddMoreTasks()) {
            BootstrapEvent next;
            if (!loadingConstraint) {
                next = iterator.next();
            } else {
                next = constraintIterator.next();
            }
            switch(next.eventType()) {
                case Database:
                    DatabaseEvent dbEvent = (DatabaseEvent) next;
                    dbTracker = new LoadDatabase(context, dbEvent, work.dbNameToLoadIn, loadTaskTracker).tasks();
                    loadTaskTracker.update(dbTracker);
                    if (work.hasDbState()) {
                        loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
                    }
                    work.updateDbEventState(dbEvent.toState());
                    scope.database = true;
                    scope.rootTasks.addAll(dbTracker.tasks());
                    dbTracker.debugLog("database");
                    break;
                case Table:
                    {
                        /*
              Implicit assumption here is that database level is processed first before table level,
              which will depend on the iterator used since it should provide the higher level directory
              listing before providing the lower level listing. This is also required such that
              the dbTracker /  tableTracker are setup correctly always.
           */
                        TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn, work.tableNameToLoadIn);
                        TableEvent tableEvent = (TableEvent) next;
                        LoadTable loadTable = new LoadTable(tableEvent, context, iterator.replLogger(), tableContext, loadTaskTracker);
                        tableTracker = loadTable.tasks();
                        if (!scope.database) {
                            scope.rootTasks.addAll(tableTracker.tasks());
                            scope.table = true;
                        }
                        setUpDependencies(dbTracker, tableTracker);
                        /*
            for table replication if we reach the max number of tasks then for the next run we will
            try to reload the same table again, this is mainly for ease of understanding the code
            as then we can avoid handling == > loading partitions for the table given that
            the creation of table lead to reaching max tasks vs,  loading next table since current
            one does not have partitions.
           */
                        // for a table we explicitly try to load partitions as there is no separate partitions events.
                        LoadPartitions loadPartitions = new LoadPartitions(context, iterator.replLogger(), loadTaskTracker, tableEvent, work.dbNameToLoadIn, tableContext);
                        TaskTracker partitionsTracker = loadPartitions.tasks();
                        partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, partitionsTracker);
                        tableTracker.debugLog("table");
                        partitionsTracker.debugLog("partitions for table");
                        break;
                    }
                case Partition:
                    {
                        /*
              This will happen only when loading tables and we reach the limit of number of tasks we can create;
              hence we know here that the table should exist and there should be a lastPartitionName
          */
                        PartitionEvent event = (PartitionEvent) next;
                        TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn, work.tableNameToLoadIn);
                        LoadPartitions loadPartitions = new LoadPartitions(context, iterator.replLogger(), tableContext, loadTaskTracker, event.asTableEvent(), work.dbNameToLoadIn, event.lastPartitionReplicated());
                        /*
               the tableTracker here should be a new instance and not an existing one as this can
               only happen when we break in between loading partitions.
           */
                        TaskTracker partitionsTracker = loadPartitions.tasks();
                        partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, partitionsTracker);
                        partitionsTracker.debugLog("partitions");
                        break;
                    }
                case Function:
                    {
                        LoadFunction loadFunction = new LoadFunction(context, iterator.replLogger(), (FunctionEvent) next, work.dbNameToLoadIn, dbTracker);
                        TaskTracker functionsTracker = loadFunction.tasks();
                        if (!scope.database) {
                            scope.rootTasks.addAll(functionsTracker.tasks());
                        } else {
                            setUpDependencies(dbTracker, functionsTracker);
                        }
                        loadTaskTracker.update(functionsTracker);
                        functionsTracker.debugLog("functions");
                        break;
                    }
                case Constraint:
                    {
                        LoadConstraint loadConstraint = new LoadConstraint(context, (ConstraintEvent) next, work.dbNameToLoadIn, dbTracker);
                        TaskTracker constraintTracker = loadConstraint.tasks();
                        scope.rootTasks.addAll(constraintTracker.tasks());
                        loadTaskTracker.update(constraintTracker);
                        constraintTracker.debugLog("constraints");
                    }
            }
            if (!loadingConstraint && !iterator.currentDbHasNext()) {
                createEndReplLogTask(context, scope, iterator.replLogger());
            }
        }
        boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState() || constraintIterator.hasNext();
        createBuilderTask(scope.rootTasks, addAnotherLoadTask);
        if (!iterator.hasNext() && !constraintIterator.hasNext()) {
            loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
            work.updateDbEventState(null);
        }
        this.childTasks = scope.rootTasks;
        /*
      Since there can be multiple rounds of this run all of which will be tied to the same
      query id -- generated in compile phase , adding a additional UUID to the end to print each run
      in separate files.
       */
        LOG.info("Root Tasks / Total Tasks : {} / {} ", childTasks.size(), loadTaskTracker.numberOfTasks());
        // Populate the driver context with the scratch dir info from the repl context, so that the temp dirs will be cleaned up later
        driverContext.getCtx().getFsScratchDirs().putAll(context.pathInfo.getFsScratchDirs());
    } catch (Exception e) {
        LOG.error("failed replication", e);
        setException(e);
        return 1;
    }
    LOG.info("completed load task run : {}", work.executedLoadTask());
    return 0;
}