[Transform] add retention policy to documentation IT (#68728)

add retention policy and sync to documentation IT, add builders for sync
and retention policy

relates #67832

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java

@@ -50,7 +50,7 @@ public class TimeRetentionPolicyConfig implements RetentionPolicyConfig {
         return PARSER.apply(parser, null);
     }
 
-    public TimeRetentionPolicyConfig(String field, TimeValue maxAge) {
+    TimeRetentionPolicyConfig(String field, TimeValue maxAge) {
         this.field = field;
         this.maxAge = maxAge;
     }
@@ -96,4 +96,37 @@ public class TimeRetentionPolicyConfig implements RetentionPolicyConfig {
     public String getName() {
         return NAME;
     }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String field;
+        private TimeValue maxAge;
+
+        /**
+         * The time field used to calculate the age of a document.
+         * @param field The field name to be used to execute the retention policy
+         * @return The {@link Builder} with the field set.
+         */
+        public Builder setField(String field) {
+            this.field = field;
+            return this;
+        }
+
+        /**
+         * The max age, all documents that are older will be deleted.
+         * @param maxAge The maximum age of a document
+         * @return The {@link Builder} with max age set.
+         */
+        public Builder setMaxAge(TimeValue maxAge) {
+            this.maxAge = maxAge;
+            return this;
+        }
+
+        public TimeRetentionPolicyConfig build() {
+            return new TimeRetentionPolicyConfig(field, maxAge);
+        }
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeSyncConfig.java

@@ -44,6 +44,8 @@ public class TimeSyncConfig implements SyncConfig {
         return PARSER.apply(parser, null);
     }
 
+    // Deprecated, the public modifier will be removed in 8.0: use the builder instead
+    @Deprecated
     public TimeSyncConfig(String field, TimeValue delay) {
         this.field = field;
         this.delay = delay;
@@ -94,4 +96,37 @@ public class TimeSyncConfig implements SyncConfig {
         return NAME;
     }
 
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String field;
+        private TimeValue delay = TimeValue.ZERO;
+
+        /**
+         * The date field that is used to identify new documents in the source.
+         * @param field The field name of the timestamp field used for synchronizing
+         * @return The {@link Builder} with the field set.
+         */
+        public Builder setField(String field) {
+            this.field = field;
+            return this;
+        }
+
+        /**
+         * The time delay between the current time and the latest input data time.
+         * The default value is 60s.
+         * @param delay the delay to use when checking for changes
+         * @return The {@link Builder} with delay set.
+         */
+        public Builder setDelay(TimeValue delay) {
+            this.delay = delay;
+            return this;
+        }
+
+        public TimeSyncConfig build() {
+            return new TimeSyncConfig(field, delay);
+        }
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java

@@ -200,7 +200,7 @@ public class TransformIT extends ESRestHighLevelClientTestCase {
 
         String id = "test-update";
         TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest").setSyncConfig(
-            new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60))
+            TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(60)).build()
         ).build();
 
         TransformClient client = highLevelClient().transform();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java

@@ -34,8 +34,11 @@ import org.elasticsearch.client.transform.UpdateTransformResponse;
 import org.elasticsearch.client.transform.transforms.DestConfig;
 import org.elasticsearch.client.transform.transforms.NodeAttributes;
 import org.elasticsearch.client.transform.transforms.QueryConfig;
+import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig;
 import org.elasticsearch.client.transform.transforms.SettingsConfig;
 import org.elasticsearch.client.transform.transforms.SourceConfig;
+import org.elasticsearch.client.transform.transforms.SyncConfig;
+import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig;
 import org.elasticsearch.client.transform.transforms.TimeSyncConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfigUpdate;
@@ -147,6 +150,18 @@ public class TransformDocumentationIT extends ESRestHighLevelClientTestCase {
             .setMaxPageSearchSize(1000) // <1>
             .build();
         // end::put-transform-settings-config
+        // tag::put-transform-retention-policy-config
+        RetentionPolicyConfig retentionPolicy = TimeRetentionPolicyConfig.builder()
+            .setField("time-field") // <1>
+            .setMaxAge(TimeValue.timeValueDays(30)) // <2>
+            .build();
+        // end::put-transform-retention-policy-config
+        // tag::put-transform-sync-config
+        SyncConfig syncConfig = TimeSyncConfig.builder()
+            .setField("time-field") // <1>
+            .setDelay(TimeValue.timeValueSeconds(30)) // <2>
+            .build();
+        // end::put-transform-sync-config
         // tag::put-transform-config
         TransformConfig transformConfig = TransformConfig
             .builder()
@@ -157,6 +172,8 @@ public class TransformDocumentationIT extends ESRestHighLevelClientTestCase {
             .setPivotConfig(pivotConfig) // <5>
             .setDescription("This is my test transform") // <6>
             .setSettings(settings) // <7>
+            .setRetentionPolicyConfig(retentionPolicy) // <8>
+            .setSyncConfig(syncConfig) // <9>
             .build();
         // end::put-transform-config
 
@@ -230,7 +247,7 @@ public class TransformDocumentationIT extends ESRestHighLevelClientTestCase {
             .setSource(SourceConfig.builder().setIndex("source-data").setQueryConfig(queryConfig).build())
             .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
-            .setSyncConfig(new TimeSyncConfig("time-field", TimeValue.timeValueSeconds(120)))
+            .setSyncConfig(TimeSyncConfig.builder().setField("time-field").setDelay(TimeValue.timeValueSeconds(120)).build())
             .build();
 
         client.transform().putTransform(new PutTransformRequest(transformConfig), RequestOptions.DEFAULT);
@@ -246,8 +263,10 @@ public class TransformDocumentationIT extends ESRestHighLevelClientTestCase {
                 .setIndex("pivot-dest")
                 .build()) // <2>
             .setFrequency(TimeValue.timeValueSeconds(15)) // <3>
-            .setSyncConfig(new TimeSyncConfig("time-field",
-                TimeValue.timeValueSeconds(120))) // <4>
+            .setSyncConfig(TimeSyncConfig.builder()
+                .setField("time-field")
+                .setDelay(TimeValue.timeValueSeconds(120))
+                .build()) // <4>
             .setDescription("This is my updated transform") // <5>
             .build();
         // end::update-transform-config

x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java

@@ -135,7 +135,7 @@ public class TransformIT extends TransformIntegTestCase {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -183,7 +183,7 @@ public class TransformIT extends TransformIntegTestCase {
         TransformConfig config =
             createTransformConfigBuilder(id, dest, QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -269,7 +269,7 @@ public class TransformIT extends TransformIntegTestCase {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -335,7 +335,7 @@ public class TransformIT extends TransformIntegTestCase {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 // set requests per second and page size low enough to fail the test if update does not succeed,
                 .setSettings(SettingsConfig.builder().setRequestsPerSecond(1F).setMaxPageSearchSize(10).build())
                 .build();

x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java

@@ -133,6 +133,6 @@ public abstract class ContinuousTestCase extends ESRestTestCase {
     }
 
     private SyncConfig getSyncConfig() {
-        return new TimeSyncConfig("timestamp", SYNC_DELAY);
+        return TimeSyncConfig.builder().setField("timestamp").setDelay(SYNC_DELAY).build();
     }
 }

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java

@@ -171,7 +171,7 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
         }
         long totalDocsWritten = totalDocsWrittenSum;
         TransformConfig config = TransformConfig.builder()
-            .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+            .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
             .setPivotConfig(PivotConfig.builder()
                 .setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("stars").field("stars")))
                 .setGroups(GroupConfig.builder().groupBy("user_id", TermsGroupSource.builder().setField("user_id").build()).build())