Examples with HoodieCommitMetadata org.apache.hudi.common.model.HoodieCommitMetadata used on opensource projects

Example 81 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class TestSimpleConcurrentFileWritesConflictResolutionStrategy method createCommit.

private void createCommit(String instantTime) throws Exception {
    String fileId1 = "file-1";
    String fileId2 = "file-2";
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    HoodieWriteStat writeStat = new HoodieWriteStat();
    writeStat.setFileId("file-1");
    commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
    commitMetadata.setOperationType(WriteOperationType.INSERT);
    HoodieTestTable.of(metaClient).addCommit(instantTime, Option.of(commitMetadata)).withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

Example 82 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class TestMetadataConversionUtils method createCompactionMetadata.

private void createCompactionMetadata(String instantTime) throws Exception {
    String fileId1 = "file-" + instantTime + "-1";
    String fileId2 = "file-" + instantTime + "-2";
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    commitMetadata.setOperationType(WriteOperationType.COMPACT);
    commitMetadata.setCompacted(true);
    HoodieTestTable.of(metaClient).addCommit(instantTime, Option.of(commitMetadata)).withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

Example 83 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class TestMetadataConversionUtils method createCommitMetadata.

private void createCommitMetadata(String instantTime) throws Exception {
    String fileId1 = "file-" + instantTime + "-1";
    String fileId2 = "file-" + instantTime + "-2";
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    commitMetadata.addMetadata("test", "test");
    commitMetadata.setOperationType(WriteOperationType.INSERT);
    HoodieTestTable.of(metaClient).addCommit(instantTime, Option.of(commitMetadata)).withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

Example 84 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class TestHoodieMetrics method testTimerCtx.

@Test
public void testTimerCtx() throws InterruptedException {
    Random rand = new Random();
    // Index metrics
    Timer.Context timer = metrics.getIndexCtx();
    // Ensure timer duration is > 0
    Thread.sleep(5);
    metrics.updateIndexMetrics("some_action", metrics.getDurationInMs(timer.stop()));
    String metricName = metrics.getMetricsName("index", "some_action.duration");
    long msec = (Long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
    assertTrue(msec > 0);
    // Rollback metrics
    timer = metrics.getRollbackCtx();
    // Ensure timer duration is > 0
    Thread.sleep(5);
    long numFilesDeleted = 1 + rand.nextInt();
    metrics.updateRollbackMetrics(metrics.getDurationInMs(timer.stop()), numFilesDeleted);
    metricName = metrics.getMetricsName("rollback", "duration");
    msec = (Long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
    assertTrue(msec > 0);
    metricName = metrics.getMetricsName("rollback", "numFilesDeleted");
    assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesDeleted);
    // Clean metrics
    timer = metrics.getRollbackCtx();
    // Ensure timer duration is > 0
    Thread.sleep(5);
    numFilesDeleted = 1 + rand.nextInt();
    metrics.updateCleanMetrics(metrics.getDurationInMs(timer.stop()), (int) numFilesDeleted);
    metricName = metrics.getMetricsName("clean", "duration");
    msec = (Long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
    assertTrue(msec > 0);
    metricName = metrics.getMetricsName("clean", "numFilesDeleted");
    assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesDeleted);
    // Finalize metrics
    timer = metrics.getFinalizeCtx();
    // Ensure timer duration is > 0
    Thread.sleep(5);
    long numFilesFinalized = 1 + rand.nextInt();
    metrics.updateFinalizeWriteMetrics(metrics.getDurationInMs(timer.stop()), (int) numFilesFinalized);
    metricName = metrics.getMetricsName("finalize", "duration");
    msec = (Long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
    assertTrue(msec > 0);
    metricName = metrics.getMetricsName("finalize", "numFilesFinalized");
    assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesFinalized);
    // Commit / deltacommit / compaction metrics
    Stream.of("commit", "deltacommit", "compaction").forEach(action -> {
        Timer.Context commitTimer = action.equals("commit") ? metrics.getCommitCtx() : action.equals("deltacommit") ? metrics.getDeltaCommitCtx() : metrics.getCompactionCtx();
        try {
            // Ensure timer duration is > 0
            Thread.sleep(5);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        long randomValue = 1 + rand.nextInt();
        HoodieCommitMetadata metadata = mock(HoodieCommitMetadata.class);
        when(metadata.fetchTotalPartitionsWritten()).thenReturn(randomValue + 1);
        when(metadata.fetchTotalFilesInsert()).thenReturn(randomValue + 2);
        when(metadata.fetchTotalFilesUpdated()).thenReturn(randomValue + 3);
        when(metadata.fetchTotalRecordsWritten()).thenReturn(randomValue + 4);
        when(metadata.fetchTotalUpdateRecordsWritten()).thenReturn(randomValue + 5);
        when(metadata.fetchTotalInsertRecordsWritten()).thenReturn(randomValue + 6);
        when(metadata.fetchTotalBytesWritten()).thenReturn(randomValue + 7);
        when(metadata.getTotalScanTime()).thenReturn(randomValue + 8);
        when(metadata.getTotalCreateTime()).thenReturn(randomValue + 9);
        when(metadata.getTotalUpsertTime()).thenReturn(randomValue + 10);
        when(metadata.getTotalCompactedRecordsUpdated()).thenReturn(randomValue + 11);
        when(metadata.getTotalLogFilesCompacted()).thenReturn(randomValue + 12);
        when(metadata.getTotalLogFilesSize()).thenReturn(randomValue + 13);
        when(metadata.getMinAndMaxEventTime()).thenReturn(Pair.of(Option.empty(), Option.empty()));
        metrics.updateCommitMetrics(randomValue + 14, commitTimer.stop(), metadata, action);
        String metricname = metrics.getMetricsName(action, "duration");
        long duration = (Long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue();
        assertTrue(duration > 0);
        metricname = metrics.getMetricsName(action, "totalPartitionsWritten");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalPartitionsWritten());
        metricname = metrics.getMetricsName(action, "totalFilesInsert");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalFilesInsert());
        metricname = metrics.getMetricsName(action, "totalFilesUpdate");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalFilesUpdated());
        metricname = metrics.getMetricsName(action, "totalRecordsWritten");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalRecordsWritten());
        metricname = metrics.getMetricsName(action, "totalUpdateRecordsWritten");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalUpdateRecordsWritten());
        metricname = metrics.getMetricsName(action, "totalInsertRecordsWritten");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalInsertRecordsWritten());
        metricname = metrics.getMetricsName(action, "totalBytesWritten");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalBytesWritten());
        metricname = metrics.getMetricsName(action, "commitTime");
        assertEquals((long) Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), randomValue + 14);
        metricname = metrics.getMetricsName(action, "totalScanTime");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalScanTime());
        metricname = metrics.getMetricsName(action, "totalCreateTime");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalCreateTime());
        metricname = metrics.getMetricsName(action, "totalUpsertTime");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalUpsertTime());
        metricname = metrics.getMetricsName(action, "totalCompactedRecordsUpdated");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalCompactedRecordsUpdated());
        metricname = metrics.getMetricsName(action, "totalLogFilesCompacted");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalLogFilesCompacted());
        metricname = metrics.getMetricsName(action, "totalLogFilesSize");
        assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalLogFilesSize());
    });
}

Example 85 with HoodieCommitMetadata

use of org.apache.hudi.common.model.HoodieCommitMetadata in project hudi by apache.

the class HoodieTestDataGenerator method createPendingReplaceFile.

public static void createPendingReplaceFile(String basePath, String instantTime, Configuration configuration) {
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    createPendingReplaceFile(basePath, instantTime, configuration, commitMetadata);
}