Examples with MultiTimingEvent org.apache.gobblin.metrics.event.MultiTimingEvent used on opensource projects

Example 1 with MultiTimingEvent

use of org.apache.gobblin.metrics.event.MultiTimingEvent in project incubator-gobblin by apache.

the class UnpartitionedTableFileSet method generateCopyEntities.

// Suppress warnings for "stepPriority++" in the PrePublishStep constructor, as stepPriority may be used later
@SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
@Override
protected Collection<CopyEntity> generateCopyEntities() throws IOException {
    MultiTimingEvent multiTimer = new MultiTimingEvent(this.helper.getEventSubmitter(), "TableCopy", true);
    int stepPriority = 0;
    String fileSet = getTable().getTableName();
    List<CopyEntity> copyEntities = Lists.newArrayList();
    Optional<Table> existingTargetTable = this.helper.getExistingTargetTable();
    if (existingTargetTable.isPresent()) {
        if (!this.helper.getTargetTable().getDataLocation().equals(existingTargetTable.get().getDataLocation())) {
            switch(this.helper.getExistingEntityPolicy()) {
                case UPDATE_TABLE:
                    // Update the location of files while keep the existing table entity.
                    log.warn("Source table will not be deregistered while file locaiton has been changed, update source table's" + " file location to" + this.helper.getTargetTable().getDataLocation());
                    existingTargetTable = Optional.absent();
                    break;
                case REPLACE_TABLE:
                case REPLACE_TABLE_AND_PARTITIONS:
                    // Required to de-register the original table.
                    log.warn("Source and target table are not compatible. Will override target table " + existingTargetTable.get().getDataLocation());
                    stepPriority = this.helper.addTableDeregisterSteps(copyEntities, fileSet, stepPriority, this.helper.getTargetTable());
                    existingTargetTable = Optional.absent();
                    break;
                default:
                    log.error("Source and target table are not compatible. Aborting copy of table " + this.helper.getTargetTable(), new HiveTableLocationNotMatchException(this.helper.getTargetTable().getDataLocation(), existingTargetTable.get().getDataLocation()));
                    multiTimer.close();
                    return Lists.newArrayList();
            }
        }
    }
    stepPriority = this.helper.addSharedSteps(copyEntities, fileSet, stepPriority);
    HiveLocationDescriptor sourceLocation = HiveLocationDescriptor.forTable(getTable(), getHiveDataset().getFs(), getHiveDataset().getProperties());
    HiveLocationDescriptor desiredTargetLocation = HiveLocationDescriptor.forTable(this.helper.getTargetTable(), this.helper.getTargetFs(), getHiveDataset().getProperties());
    Optional<HiveLocationDescriptor> existingTargetLocation = existingTargetTable.isPresent() ? Optional.of(HiveLocationDescriptor.forTable(existingTargetTable.get(), this.helper.getTargetFs(), getHiveDataset().getProperties())) : Optional.<HiveLocationDescriptor>absent();
    if (this.helper.getFastTableSkip().isPresent() && this.helper.getFastTableSkip().get().apply(this.helper)) {
        log.info(String.format("Skipping copy of table %s due to fast table skip predicate.", getTable().getDbName() + "." + getTable().getTableName()));
        multiTimer.close();
        return Lists.newArrayList();
    }
    HiveCopyEntityHelper.DiffPathSet diffPathSet = HiveCopyEntityHelper.fullPathDiff(sourceLocation, desiredTargetLocation, existingTargetLocation, Optional.<Partition>absent(), multiTimer, this.helper);
    multiTimer.nextStage(HiveCopyEntityHelper.Stages.FULL_PATH_DIFF);
    // Could used to delete files for the existing snapshot
    DeleteFileCommitStep deleteStep = DeleteFileCommitStep.fromPaths(this.helper.getTargetFs(), diffPathSet.pathsToDelete, getHiveDataset().getProperties());
    copyEntities.add(new PrePublishStep(fileSet, Maps.<String, String>newHashMap(), deleteStep, stepPriority++));
    for (CopyableFile.Builder builder : this.helper.getCopyableFilesFromPaths(diffPathSet.filesToCopy, this.helper.getConfiguration(), Optional.<Partition>absent())) {
        CopyableFile fileEntity = builder.fileSet(fileSet).datasetOutputPath(desiredTargetLocation.location.toString()).build();
        this.helper.setCopyableFileDatasets(fileEntity);
        copyEntities.add(fileEntity);
    }
    multiTimer.close();
    return copyEntities;
}

Example 2 with MultiTimingEvent

use of org.apache.gobblin.metrics.event.MultiTimingEvent in project incubator-gobblin by apache.

the class HivePartitionFileSet method generateCopyEntities.

@Override
protected Collection<CopyEntity> generateCopyEntities() throws IOException {
    try (Closer closer = Closer.create()) {
        MultiTimingEvent multiTimer = closer.register(new MultiTimingEvent(this.eventSubmitter, "PartitionCopy", true));
        int stepPriority = 0;
        String fileSet = HiveCopyEntityHelper.gson.toJson(this.partition.getValues());
        List<CopyEntity> copyEntities = Lists.newArrayList();
        stepPriority = hiveCopyEntityHelper.addSharedSteps(copyEntities, fileSet, stepPriority);
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.COMPUTE_TARGETS);
        Path targetPath = hiveCopyEntityHelper.getTargetLocation(hiveCopyEntityHelper.getDataset().fs, hiveCopyEntityHelper.getTargetFs(), this.partition.getDataLocation(), Optional.of(this.partition));
        Partition targetPartition = getTargetPartition(this.partition, targetPath);
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.EXISTING_PARTITION);
        if (this.existingTargetPartition.isPresent()) {
            hiveCopyEntityHelper.getTargetPartitions().remove(this.partition.getValues());
            try {
                checkPartitionCompatibility(targetPartition, this.existingTargetPartition.get());
            } catch (IOException ioe) {
                if (hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_PARTITIONS && hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_TABLE_AND_PARTITIONS) {
                    log.error("Source and target partitions are not compatible. Aborting copy of partition " + this.partition, ioe);
                    return Lists.newArrayList();
                }
                log.warn("Source and target partitions are not compatible. Will override target partition: " + ioe.getMessage());
                log.debug("Incompatibility details: ", ioe);
                stepPriority = hiveCopyEntityHelper.addPartitionDeregisterSteps(copyEntities, fileSet, stepPriority, hiveCopyEntityHelper.getTargetTable(), this.existingTargetPartition.get());
                this.existingTargetPartition = Optional.absent();
            }
        }
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.PARTITION_SKIP_PREDICATE);
        if (hiveCopyEntityHelper.getFastPartitionSkip().isPresent() && hiveCopyEntityHelper.getFastPartitionSkip().get().apply(this)) {
            log.info(String.format("Skipping copy of partition %s due to fast partition skip predicate.", this.partition.getCompleteName()));
            return Lists.newArrayList();
        }
        HiveSpec partitionHiveSpec = new SimpleHiveSpec.Builder<>(targetPath).withTable(HiveMetaStoreUtils.getHiveTable(hiveCopyEntityHelper.getTargetTable().getTTable())).withPartition(Optional.of(HiveMetaStoreUtils.getHivePartition(targetPartition.getTPartition()))).build();
        HiveRegisterStep register = new HiveRegisterStep(hiveCopyEntityHelper.getTargetURI(), partitionHiveSpec, hiveCopyEntityHelper.getHiveRegProps());
        copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), register, stepPriority++));
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.CREATE_LOCATIONS);
        HiveLocationDescriptor sourceLocation = HiveLocationDescriptor.forPartition(this.partition, hiveCopyEntityHelper.getDataset().fs, this.properties);
        HiveLocationDescriptor desiredTargetLocation = HiveLocationDescriptor.forPartition(targetPartition, hiveCopyEntityHelper.getTargetFs(), this.properties);
        Optional<HiveLocationDescriptor> existingTargetLocation = this.existingTargetPartition.isPresent() ? Optional.of(HiveLocationDescriptor.forPartition(this.existingTargetPartition.get(), hiveCopyEntityHelper.getTargetFs(), this.properties)) : Optional.<HiveLocationDescriptor>absent();
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.FULL_PATH_DIFF);
        HiveCopyEntityHelper.DiffPathSet diffPathSet = HiveCopyEntityHelper.fullPathDiff(sourceLocation, desiredTargetLocation, existingTargetLocation, Optional.<Partition>absent(), multiTimer, hiveCopyEntityHelper);
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.CREATE_DELETE_UNITS);
        if (diffPathSet.pathsToDelete.size() > 0) {
            DeleteFileCommitStep deleteStep = DeleteFileCommitStep.fromPaths(hiveCopyEntityHelper.getTargetFs(), diffPathSet.pathsToDelete, hiveCopyEntityHelper.getDataset().properties);
            copyEntities.add(new PrePublishStep(fileSet, Maps.<String, String>newHashMap(), deleteStep, stepPriority++));
        }
        multiTimer.nextStage(HiveCopyEntityHelper.Stages.CREATE_COPY_UNITS);
        for (CopyableFile.Builder builder : hiveCopyEntityHelper.getCopyableFilesFromPaths(diffPathSet.filesToCopy, hiveCopyEntityHelper.getConfiguration(), Optional.of(this.partition))) {
            CopyableFile fileEntity = builder.fileSet(fileSet).checksum(new byte[0]).datasetOutputPath(desiredTargetLocation.location.toString()).build();
            this.hiveCopyEntityHelper.setCopyableFileDatasets(fileEntity);
            copyEntities.add(fileEntity);
        }
        log.info("Created {} copy entities for partition {}", copyEntities.size(), this.partition.getCompleteName());
        return copyEntities;
    }
}

Example 3 with MultiTimingEvent

use of org.apache.gobblin.metrics.event.MultiTimingEvent in project incubator-gobblin by apache.

the class HiveCopyEntityHelperTest method testFullPathDiffWithUnmanagedPathsWithoutDeletePolicy.

@Test
public void testFullPathDiffWithUnmanagedPathsWithoutDeletePolicy() throws Exception {
    Map<Path, FileStatus> sourceMap = Maps.newHashMap();
    Map<Path, FileStatus> targetDesiredMap = Maps.newHashMap();
    List<Path> expectedFilesToCopy = Lists.newArrayList();
    List<Path> expectedFilesToSkipCopy = Lists.newArrayList();
    List<Path> expectedFilesToDelete = Lists.newArrayList();
    List<Path> expectedFilesToSkipDelete = Lists.newArrayList();
    populateSourceAndTargetEntities(sourceMap, targetDesiredMap, expectedFilesToCopy, expectedFilesToSkipCopy, expectedFilesToDelete, expectedFilesToSkipDelete);
    // add un-managed files to the target path
    Path path6 = new Path("path6");
    Path targetPath6 = new Path(targetRoot, path6);
    Map<Path, FileStatus> targetDesiredMapWithExtraFile = Maps.newHashMap(targetDesiredMap);
    targetDesiredMapWithExtraFile.put(targetPath6, getFileStatus(targetPath6, 0, 10));
    expectedFilesToDelete.add(targetPath6);
    TestLocationDescriptor sourceLocation = new TestLocationDescriptor(sourceMap);
    TestLocationDescriptor targetDesiredLocation = new TestLocationDescriptor(targetDesiredMapWithExtraFile);
    TestLocationDescriptor existingTargetLocation = new TestLocationDescriptor(Maps.newHashMap(targetDesiredMap));
    Table table = Mockito.mock(Table.class);
    HiveDataset hiveDataset = Mockito.mock(HiveDataset.class);
    MultiTimingEvent timer = Mockito.mock(MultiTimingEvent.class);
    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
    HiveTargetPathHelper targetPathHelper = Mockito.mock(HiveTargetPathHelper.class);
    Mockito.when(helper.getDataset()).thenReturn(hiveDataset);
    Mockito.when(hiveDataset.getTable()).thenReturn(table);
    Mockito.when(table.getCompleteName()).thenReturn("table1");
    Mockito.when(targetPathHelper.getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() {

        @Override
        public Path answer(InvocationOnMock invocation) throws Throwable {
            Path path = (Path) invocation.getArguments()[0];
            return new Path(path.toString().replace(sourceRoot.toString(), targetRoot.toString()));
        }
    });
    Mockito.when(helper.getTargetPathHelper()).thenReturn(targetPathHelper);
    // Add policy to not delete un-managed data
    Mockito.when(helper.getUnmanagedDataPolicy()).thenReturn(HiveCopyEntityHelper.UnmanagedDataPolicy.ABORT);
    // We should receive an exception that un-managed files are detected
    try {
        HiveCopyEntityHelper.DiffPathSet diff = HiveCopyEntityHelper.fullPathDiff(sourceLocation, targetDesiredLocation, Optional.<HiveLocationDescriptor>of(existingTargetLocation), Optional.<Partition>absent(), timer, helper);
        Assert.fail("Expected an IOException but did not receive any");
    } catch (IOException ex) {
        // Ignore IOException if message is what we expect
        String expectedExceptionMessage = "New table / partition would pick up existing, undesired files in target file " + "system. table1, files [/target/path6].";
        Assert.assertEquals(ex.getMessage(), expectedExceptionMessage);
    }
}

Example 4 with MultiTimingEvent

use of org.apache.gobblin.metrics.event.MultiTimingEvent in project incubator-gobblin by apache.

the class HiveCopyEntityHelperTest method testFullPathDiffWithUnmanagedPathsWithDeletePolicy.

@Test
public void testFullPathDiffWithUnmanagedPathsWithDeletePolicy() throws Exception {
    Map<Path, FileStatus> sourceMap = Maps.newHashMap();
    Map<Path, FileStatus> targetDesiredMap = Maps.newHashMap();
    List<Path> expectedFilesToCopy = Lists.newArrayList();
    List<Path> expectedFilesToSkipCopy = Lists.newArrayList();
    List<Path> expectedFilesToDelete = Lists.newArrayList();
    List<Path> expectedFilesToSkipDelete = Lists.newArrayList();
    populateSourceAndTargetEntities(sourceMap, targetDesiredMap, expectedFilesToCopy, expectedFilesToSkipCopy, expectedFilesToDelete, expectedFilesToSkipDelete);
    // add un-managed files to the target path
    Path path6 = new Path("path6");
    Path targetPath6 = new Path(targetRoot, path6);
    Map<Path, FileStatus> targetDesiredMapWithExtraFile = Maps.newHashMap(targetDesiredMap);
    targetDesiredMapWithExtraFile.put(targetPath6, getFileStatus(targetPath6, 0, 10));
    expectedFilesToDelete.add(targetPath6);
    TestLocationDescriptor sourceLocation = new TestLocationDescriptor(sourceMap);
    TestLocationDescriptor targetDesiredLocation = new TestLocationDescriptor(targetDesiredMapWithExtraFile);
    TestLocationDescriptor existingTargetLocation = new TestLocationDescriptor(Maps.newHashMap(targetDesiredMap));
    Table table = Mockito.mock(Table.class);
    HiveDataset hiveDataset = Mockito.mock(HiveDataset.class);
    MultiTimingEvent timer = Mockito.mock(MultiTimingEvent.class);
    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
    HiveTargetPathHelper targetPathHelper = Mockito.mock(HiveTargetPathHelper.class);
    Mockito.when(helper.getDataset()).thenReturn(hiveDataset);
    Mockito.when(hiveDataset.getTable()).thenReturn(table);
    Mockito.when(table.getCompleteName()).thenReturn("table1");
    Mockito.when(targetPathHelper.getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() {

        @Override
        public Path answer(InvocationOnMock invocation) throws Throwable {
            Path path = (Path) invocation.getArguments()[0];
            return new Path(path.toString().replace(sourceRoot.toString(), targetRoot.toString()));
        }
    });
    Mockito.when(helper.getTargetPathHelper()).thenReturn(targetPathHelper);
    // Add policy to delete un-managed data
    Mockito.when(helper.getUnmanagedDataPolicy()).thenReturn(HiveCopyEntityHelper.UnmanagedDataPolicy.DELETE_UNMANAGED_DATA);
    // Since policy is specified to delete un-managed data, this should not throw exception and un-managed file should
    // .. show up in pathsToDelete in the diff
    HiveCopyEntityHelper.DiffPathSet diff = HiveCopyEntityHelper.fullPathDiff(sourceLocation, targetDesiredLocation, Optional.<HiveLocationDescriptor>of(existingTargetLocation), Optional.<Partition>absent(), timer, helper);
    Assert.assertEquals(diff.filesToCopy.size(), expectedFilesToCopy.size());
    for (Path expectedFileToCopy : expectedFilesToCopy) {
        Assert.assertTrue(containsPath(diff.filesToCopy, expectedFileToCopy));
    }
    for (Path expectedFileToSkipCopy : expectedFilesToSkipCopy) {
        Assert.assertFalse(containsPath(diff.filesToCopy, expectedFileToSkipCopy));
    }
    Assert.assertEquals(diff.pathsToDelete.size(), expectedFilesToDelete.size());
    for (Path expectedFileToDelete : expectedFilesToDelete) {
        Assert.assertTrue(diff.pathsToDelete.contains(expectedFileToDelete));
    }
    for (Path expectedFileToSkipDelete : expectedFilesToSkipDelete) {
        Assert.assertFalse(diff.pathsToDelete.contains(expectedFileToSkipDelete));
    }
}

Example 5 with MultiTimingEvent

use of org.apache.gobblin.metrics.event.MultiTimingEvent in project incubator-gobblin by apache.

the class HiveCopyEntityHelperTest method testFullPathDiff.

@Test
public void testFullPathDiff() throws Exception {
    Map<Path, FileStatus> sourceMap = Maps.newHashMap();
    Map<Path, FileStatus> targetDesiredMap = Maps.newHashMap();
    List<Path> expectedFilesToCopy = Lists.newArrayList();
    List<Path> expectedFilesToSkipCopy = Lists.newArrayList();
    List<Path> expectedFilesToDelete = Lists.newArrayList();
    List<Path> expectedFilesToSkipDelete = Lists.newArrayList();
    populateSourceAndTargetEntities(sourceMap, targetDesiredMap, expectedFilesToCopy, expectedFilesToSkipCopy, expectedFilesToDelete, expectedFilesToSkipDelete);
    TestLocationDescriptor sourceLocation = new TestLocationDescriptor(sourceMap);
    TestLocationDescriptor targetDesiredLocation = new TestLocationDescriptor(targetDesiredMap);
    TestLocationDescriptor existingTargetLocation = new TestLocationDescriptor(Maps.newHashMap(targetDesiredMap));
    MultiTimingEvent timer = Mockito.mock(MultiTimingEvent.class);
    HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
    HiveTargetPathHelper targetPathHelper = Mockito.mock(HiveTargetPathHelper.class);
    Mockito.when(targetPathHelper.getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() {

        @Override
        public Path answer(InvocationOnMock invocation) throws Throwable {
            Path path = (Path) invocation.getArguments()[0];
            return new Path(path.toString().replace(sourceRoot.toString(), targetRoot.toString()));
        }
    });
    Mockito.when(helper.getTargetPathHelper()).thenReturn(targetPathHelper);
    HiveCopyEntityHelper.DiffPathSet diff = HiveCopyEntityHelper.fullPathDiff(sourceLocation, targetDesiredLocation, Optional.<HiveLocationDescriptor>of(existingTargetLocation), Optional.<Partition>absent(), timer, helper);
    Assert.assertEquals(diff.filesToCopy.size(), expectedFilesToCopy.size());
    for (Path expectedFileToCopy : expectedFilesToCopy) {
        Assert.assertTrue(containsPath(diff.filesToCopy, expectedFileToCopy));
    }
    for (Path expectedFileToSkipCopy : expectedFilesToSkipCopy) {
        Assert.assertFalse(containsPath(diff.filesToCopy, expectedFileToSkipCopy));
    }
    Assert.assertEquals(diff.pathsToDelete.size(), expectedFilesToDelete.size());
    for (Path expectedFileToDelete : expectedFilesToDelete) {
        Assert.assertTrue(diff.pathsToDelete.contains(expectedFileToDelete));
    }
    for (Path expectedFileToSkipDelete : expectedFilesToSkipDelete) {
        Assert.assertFalse(diff.pathsToDelete.contains(expectedFileToSkipDelete));
    }
}