Examples with DataSourceMetadata io.druid.indexing.overlord.DataSourceMetadata used on opensource projects

Example 1 with DataSourceMetadata

use of io.druid.indexing.overlord.DataSourceMetadata in project druid by druid-io.

the class KafkaSupervisor method resetInternal.

@VisibleForTesting
void resetInternal(DataSourceMetadata dataSourceMetadata) {
    if (dataSourceMetadata == null) {
        // Reset everything
        boolean result = indexerMetadataStorageCoordinator.deleteDataSourceMetadata(dataSource);
        log.info("Reset dataSource[%s] - dataSource metadata entry deleted? [%s]", dataSource, result);
        killTaskGroupForPartitions(JavaCompatUtils.keySet(taskGroups));
    } else if (!(dataSourceMetadata instanceof KafkaDataSourceMetadata)) {
        throw new IAE("Expected KafkaDataSourceMetadata but found instance of [%s]", dataSourceMetadata.getClass());
    } else {
        // Reset only the partitions in dataSourceMetadata if it has not been reset yet
        final KafkaDataSourceMetadata resetKafkaMetadata = (KafkaDataSourceMetadata) dataSourceMetadata;
        if (resetKafkaMetadata.getKafkaPartitions().getTopic().equals(ioConfig.getTopic())) {
            // metadata can be null
            final DataSourceMetadata metadata = indexerMetadataStorageCoordinator.getDataSourceMetadata(dataSource);
            if (metadata != null && !(metadata instanceof KafkaDataSourceMetadata)) {
                throw new IAE("Expected KafkaDataSourceMetadata from metadata store but found instance of [%s]", metadata.getClass());
            }
            final KafkaDataSourceMetadata currentMetadata = (KafkaDataSourceMetadata) metadata;
            // defend against consecutive reset requests from replicas
            // as well as the case where the metadata store do not have an entry for the reset partitions
            boolean doReset = false;
            for (Map.Entry<Integer, Long> resetPartitionOffset : resetKafkaMetadata.getKafkaPartitions().getPartitionOffsetMap().entrySet()) {
                final Long partitionOffsetInMetadataStore = currentMetadata == null ? null : currentMetadata.getKafkaPartitions().getPartitionOffsetMap().get(resetPartitionOffset.getKey());
                final TaskGroup partitionTaskGroup = taskGroups.get(getTaskGroupIdForPartition(resetPartitionOffset.getKey()));
                if (partitionOffsetInMetadataStore != null || (partitionTaskGroup != null && partitionTaskGroup.partitionOffsets.get(resetPartitionOffset.getKey()).equals(resetPartitionOffset.getValue()))) {
                    doReset = true;
                    break;
                }
            }
            if (!doReset) {
                return;
            }
            boolean metadataUpdateSuccess = false;
            if (currentMetadata == null) {
                metadataUpdateSuccess = true;
            } else {
                final DataSourceMetadata newMetadata = currentMetadata.minus(resetKafkaMetadata);
                try {
                    metadataUpdateSuccess = indexerMetadataStorageCoordinator.resetDataSourceMetadata(dataSource, newMetadata);
                } catch (IOException e) {
                    log.error("Resetting DataSourceMetadata failed [%s]", e.getMessage());
                    Throwables.propagate(e);
                }
            }
            if (metadataUpdateSuccess) {
                killTaskGroupForPartitions(JavaCompatUtils.keySet(resetKafkaMetadata.getKafkaPartitions().getPartitionOffsetMap()));
            } else {
                throw new ISE("Unable to reset metadata");
            }
        } else {
            log.warn("Reset metadata topic [%s] and supervisor's topic [%s] do not match", resetKafkaMetadata.getKafkaPartitions().getTopic(), ioConfig.getTopic());
        }
    }
}

Example 2 with DataSourceMetadata

use of io.druid.indexing.overlord.DataSourceMetadata in project druid by druid-io.

the class KafkaSupervisorTest method testResetDataSourceMetadata.

@Test
public void testResetDataSourceMetadata() throws Exception {
    supervisor = getSupervisor(1, 1, true, "PT1H", null);
    expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    expect(taskRunner.getRunningTasks()).andReturn(Collections.EMPTY_LIST).anyTimes();
    expect(taskStorage.getActiveTasks()).andReturn(ImmutableList.<Task>of()).anyTimes();
    taskRunner.registerListener(anyObject(TaskRunnerListener.class), anyObject(Executor.class));
    replayAll();
    supervisor.start();
    supervisor.runInternal();
    verifyAll();
    Capture<String> captureDataSource = EasyMock.newCapture();
    Capture<DataSourceMetadata> captureDataSourceMetadata = EasyMock.newCapture();
    KafkaDataSourceMetadata kafkaDataSourceMetadata = new KafkaDataSourceMetadata(new KafkaPartitions(KAFKA_TOPIC, ImmutableMap.of(0, 1000L, 1, 1000L, 2, 1000L)));
    KafkaDataSourceMetadata resetMetadata = new KafkaDataSourceMetadata(new KafkaPartitions(KAFKA_TOPIC, ImmutableMap.of(1, 1000L, 2, 1000L)));
    KafkaDataSourceMetadata expectedMetadata = new KafkaDataSourceMetadata(new KafkaPartitions(KAFKA_TOPIC, ImmutableMap.of(0, 1000L)));
    reset(indexerMetadataStorageCoordinator);
    expect(indexerMetadataStorageCoordinator.getDataSourceMetadata(DATASOURCE)).andReturn(kafkaDataSourceMetadata);
    expect(indexerMetadataStorageCoordinator.resetDataSourceMetadata(EasyMock.capture(captureDataSource), EasyMock.capture(captureDataSourceMetadata))).andReturn(true);
    replay(indexerMetadataStorageCoordinator);
    supervisor.resetInternal(resetMetadata);
    verifyAll();
    Assert.assertEquals(captureDataSource.getValue(), DATASOURCE);
    Assert.assertEquals(captureDataSourceMetadata.getValue(), expectedMetadata);
}

Example 3 with DataSourceMetadata

use of io.druid.indexing.overlord.DataSourceMetadata in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinator method updateDataSourceMetadataWithHandle.

/**
   * Compare-and-swap dataSource metadata in a transaction. This will only modify dataSource metadata if it equals
   * oldCommitMetadata when this function is called (based on T.equals). This method is idempotent in that if
   * the metadata already equals newCommitMetadata, it will return true.
   *
   * @param handle        database handle
   * @param dataSource    druid dataSource
   * @param startMetadata dataSource metadata pre-insert must match this startMetadata according to
   *                      {@link DataSourceMetadata#matches(DataSourceMetadata)}
   * @param endMetadata   dataSource metadata post-insert will have this endMetadata merged in with
   *                      {@link DataSourceMetadata#plus(DataSourceMetadata)}
   *
   * @return true if dataSource metadata was updated from matching startMetadata to matching endMetadata
   */
protected DataSourceMetadataUpdateResult updateDataSourceMetadataWithHandle(final Handle handle, final String dataSource, final DataSourceMetadata startMetadata, final DataSourceMetadata endMetadata) throws IOException {
    Preconditions.checkNotNull(dataSource, "dataSource");
    Preconditions.checkNotNull(startMetadata, "startMetadata");
    Preconditions.checkNotNull(endMetadata, "endMetadata");
    final byte[] oldCommitMetadataBytesFromDb = getDataSourceMetadataWithHandleAsBytes(handle, dataSource);
    final String oldCommitMetadataSha1FromDb;
    final DataSourceMetadata oldCommitMetadataFromDb;
    if (oldCommitMetadataBytesFromDb == null) {
        oldCommitMetadataSha1FromDb = null;
        oldCommitMetadataFromDb = null;
    } else {
        oldCommitMetadataSha1FromDb = BaseEncoding.base16().encode(Hashing.sha1().hashBytes(oldCommitMetadataBytesFromDb).asBytes());
        oldCommitMetadataFromDb = jsonMapper.readValue(oldCommitMetadataBytesFromDb, DataSourceMetadata.class);
    }
    final boolean startMetadataMatchesExisting = oldCommitMetadataFromDb == null ? startMetadata.isValidStart() : startMetadata.matches(oldCommitMetadataFromDb);
    if (!startMetadataMatchesExisting) {
        // Not in the desired start state.
        log.info("Not updating metadata, existing state is not the expected start state.");
        return DataSourceMetadataUpdateResult.FAILURE;
    }
    final DataSourceMetadata newCommitMetadata = oldCommitMetadataFromDb == null ? endMetadata : oldCommitMetadataFromDb.plus(endMetadata);
    final byte[] newCommitMetadataBytes = jsonMapper.writeValueAsBytes(newCommitMetadata);
    final String newCommitMetadataSha1 = BaseEncoding.base16().encode(Hashing.sha1().hashBytes(newCommitMetadataBytes).asBytes());
    final DataSourceMetadataUpdateResult retVal;
    if (oldCommitMetadataBytesFromDb == null) {
        // SELECT -> INSERT can fail due to races; callers must be prepared to retry.
        final int numRows = handle.createStatement(String.format("INSERT INTO %s (dataSource, created_date, commit_metadata_payload, commit_metadata_sha1) " + "VALUES (:dataSource, :created_date, :commit_metadata_payload, :commit_metadata_sha1)", dbTables.getDataSourceTable())).bind("dataSource", dataSource).bind("created_date", new DateTime().toString()).bind("commit_metadata_payload", newCommitMetadataBytes).bind("commit_metadata_sha1", newCommitMetadataSha1).execute();
        retVal = numRows == 1 ? DataSourceMetadataUpdateResult.SUCCESS : DataSourceMetadataUpdateResult.TRY_AGAIN;
    } else {
        // Expecting a particular old metadata; use the SHA1 in a compare-and-swap UPDATE
        final int numRows = handle.createStatement(String.format("UPDATE %s SET " + "commit_metadata_payload = :new_commit_metadata_payload, " + "commit_metadata_sha1 = :new_commit_metadata_sha1 " + "WHERE dataSource = :dataSource AND commit_metadata_sha1 = :old_commit_metadata_sha1", dbTables.getDataSourceTable())).bind("dataSource", dataSource).bind("old_commit_metadata_sha1", oldCommitMetadataSha1FromDb).bind("new_commit_metadata_payload", newCommitMetadataBytes).bind("new_commit_metadata_sha1", newCommitMetadataSha1).execute();
        retVal = numRows == 1 ? DataSourceMetadataUpdateResult.SUCCESS : DataSourceMetadataUpdateResult.TRY_AGAIN;
    }
    if (retVal == DataSourceMetadataUpdateResult.SUCCESS) {
        log.info("Updated metadata from[%s] to[%s].", oldCommitMetadataFromDb, newCommitMetadata);
    } else {
        log.info("Not updating metadata, compare-and-swap failure.");
    }
    return retVal;
}

Example 4 with DataSourceMetadata

use of io.druid.indexing.overlord.DataSourceMetadata in project druid by druid-io.

the class IndexerSQLMetadataStorageCoordinatorTest method testTransactionalAnnounceRetryAndSuccess.

@Test
public void testTransactionalAnnounceRetryAndSuccess() throws IOException {
    final AtomicLong attemptCounter = new AtomicLong();
    final IndexerSQLMetadataStorageCoordinator failOnceCoordinator = new IndexerSQLMetadataStorageCoordinator(mapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector) {

        @Override
        protected DataSourceMetadataUpdateResult updateDataSourceMetadataWithHandle(Handle handle, String dataSource, DataSourceMetadata startMetadata, DataSourceMetadata endMetadata) throws IOException {
            metadataUpdateCounter.getAndIncrement();
            if (attemptCounter.getAndIncrement() == 0) {
                return DataSourceMetadataUpdateResult.TRY_AGAIN;
            } else {
                return super.updateDataSourceMetadataWithHandle(handle, dataSource, startMetadata, endMetadata);
            }
        }
    };
    // Insert first segment.
    final SegmentPublishResult result1 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment), new ObjectMetadata(null), new ObjectMetadata(ImmutableMap.of("foo", "bar")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment), true), result1);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment.getIdentifier()));
    // Reset attempt counter to induce another failure.
    attemptCounter.set(0);
    // Insert second segment.
    final SegmentPublishResult result2 = failOnceCoordinator.announceHistoricalSegments(ImmutableSet.of(defaultSegment2), new ObjectMetadata(ImmutableMap.of("foo", "bar")), new ObjectMetadata(ImmutableMap.of("foo", "baz")));
    Assert.assertEquals(new SegmentPublishResult(ImmutableSet.of(defaultSegment2), true), result2);
    Assert.assertArrayEquals(mapper.writeValueAsString(defaultSegment2).getBytes("UTF-8"), derbyConnector.lookup(derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), "id", "payload", defaultSegment2.getIdentifier()));
    // Examine metadata.
    Assert.assertEquals(new ObjectMetadata(ImmutableMap.of("foo", "baz")), failOnceCoordinator.getDataSourceMetadata("fooDataSource"));
    // Should be tried twice per call.
    Assert.assertEquals(4, metadataUpdateCounter.get());
}