Examples with Pipeline com.hazelcast.jet.pipeline.Pipeline used on opensource projects

Example 76 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class AuthElasticSourcesTest method given_clientWithoutAuthentication_whenReadFromElasticSource_then_failWithAuthenticationException.

@Test
public void given_clientWithoutAuthentication_whenReadFromElasticSource_then_failWithAuthenticationException() {
    ElasticsearchContainer container = ElasticSupport.secureElastic.get();
    String containerIp = container.getContainerIpAddress();
    Integer port = container.getMappedPort(PORT);
    Pipeline p = Pipeline.create();
    p.readFrom(elasticSource(() -> client(containerIp, port))).writeTo(Sinks.list(results));
    assertThatThrownBy(() -> submitJob(p)).hasRootCauseInstanceOf(ResponseException.class).hasStackTraceContaining("missing authentication credentials");
}

Example 77 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class AuthElasticSourcesTest method given_clientWithWrongPassword_whenReadFromElasticSource_thenFailWithAuthenticationException.

@Test
public void given_clientWithWrongPassword_whenReadFromElasticSource_thenFailWithAuthenticationException() {
    ElasticsearchContainer container = ElasticSupport.secureElastic.get();
    String containerIp = container.getContainerIpAddress();
    Integer port = container.getMappedPort(PORT);
    Pipeline p = Pipeline.create();
    p.readFrom(elasticSource(() -> client("elastic", "WrongPassword", containerIp, port))).writeTo(Sinks.list(results));
    assertThatThrownBy(() -> submitJob(p)).hasRootCauseInstanceOf(ResponseException.class).hasStackTraceContaining("unable to authenticate user [elastic]");
}

Example 78 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class CommonElasticSinksTest method given_documentInIndex_whenWriteToElasticSinkDeleteRequestTwice_then_jobShouldFinishSuccessfully.

@Test
public void given_documentInIndex_whenWriteToElasticSinkDeleteRequestTwice_then_jobShouldFinishSuccessfully() throws Exception {
    Map<String, Object> doc = new HashMap<>();
    doc.put("name", "Frantisek");
    String id = indexDocument("my-index", doc);
    Sink<String> elasticSink = new ElasticSinkBuilder<>().clientFn(elasticClientSupplier()).mapToRequestFn((String item) -> new DeleteRequest("my-index", item)).bulkRequestFn(() -> new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)).build();
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items(id)).writeTo(elasticSink);
    // Submit job 2x to delete non-existing document on 2nd run
    submitJob(p);
    submitJob(p);
    assertNoDocuments("my-index");
}

Example 79 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class CommonElasticSinksTest method given_documentInIndex_whenWriteToElasticSinkDeleteRequest_then_documentIsDeleted.

@Test
public void given_documentInIndex_whenWriteToElasticSinkDeleteRequest_then_documentIsDeleted() throws Exception {
    Map<String, Object> doc = new HashMap<>();
    doc.put("name", "Fra");
    String id = indexDocument("my-index", doc);
    Sink<TestItem> elasticSink = ElasticSinks.elastic(elasticClientSupplier(), (item) -> new DeleteRequest("my-index", item.getId()));
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items(new TestItem(id, "Frantisek"))).writeTo(elasticSink);
    submitJob(p);
    refreshIndex();
    assertNoDocuments("my-index");
}

Example 80 with Pipeline

use of com.hazelcast.jet.pipeline.Pipeline in project hazelcast by hazelcast.

the class MySqlCdcIntegrationTest method restart.

@Test
@Category(NightlyTest.class)
public void restart() throws Exception {
    // given
    List<String> expectedRecords = Arrays.asList("1004/1:UPDATE:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005/0:INSERT:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}", "1005/1:DELETE:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}");
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE);
    Job job = hz.getJet().newJob(pipeline, jobConfig);
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    assertEqualsEventually(() -> hz.getMap("results").size(), 4);
    // then
    hz.getMap("results").destroy();
    // when
    assertEqualsEventually(() -> hz.getMap("results").size(), 0);
    // then
    job.restart();
    // when
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    // then update a record
    try (Connection connection = getConnection(mysql, "inventory")) {
        Statement statement = connection.createStatement();
        statement.addBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004");
        statement.addBatch("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        statement.addBatch("DELETE FROM customers WHERE id=1005");
        statement.executeBatch();
    }
    // then
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}