Add initial esql test benchmark tests (#126224)

* Add initial esql test benchmark tests
* Fix build-benchmark pipeline
* Update gredle profiler

.buildkite/pipelines/pull-request/build-benchmark.yml

@@ -7,16 +7,19 @@ steps:
     command: |
       .ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
       .ci/scripts/install-gradle-profiler.sh
-      .ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-{{matrix.part}}.scenarios --project-dir . --output-dir profile-out
-      mkdir build
-      tar -czf build/$BUILDKITE_BUILD_NUMBER.tar.bz2 profile-out
+      .ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-{{matrix.part}}.scenarios --measure-config-time --project-dir . --output-dir profile-out
+      mkdir -p build
+      buildkite-agent artifact upload "profile-out/*"
+      cat << EOF | buildkite-agent annotate --style "info"
+      Benchmark Html Report <a href="artifact://profile-out/benchmark.html">Benchmark Html Report</a>
+      EOF
     matrix:
       setup:
         part:
-          - part1
-          - part2
+          - esql-tests
     env:
       BUILD_PERFORMANCE_TEST: "true"
+
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004

.ci/scripts/install-gradle-profiler.sh

@@ -2,7 +2,7 @@
 
 set -e
 # profiler version we wanna install
-PROFILER_VERSION="0.16.0"
-wget https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
+PROFILER_VERSION="0.22.0"
+wget https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
 unzip $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
 mv $WORKSPACE/gradle-profiler-$PROFILER_VERSION $WORKSPACE/gradle-profiler

build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios

@@ -1,72 +0,0 @@
-// ensure branch scenario is listed first as this is the gradle version that will picked for inspecting the build
-default-scenarios = ["buildConfiguration_branch", "buildConfiguration_main", "single_project_branch", "single_project_main"]
-
-buildConfiguration_main {
-    title = "configuration phase (main)"
-    versions = ["@mainWrapper@"]
-    tasks = ["help"]
-    gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "main"
-    }
-}
-
-buildConfiguration_branch {
-    title = "configuration phase (@testGitCommit@)"
-    versions = ["@branchWrapper@"]
-    tasks = ["help"]
-    gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "@testGitCommit@"
-    }
-}
-
-single_project_main {
-    title = "single project (main)"
-    versions = ["@mainWrapper@"]
-    tasks = [":server:precommit"]
-    gradle-args = ["--no-scan", "--stacktrace"]
-    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "main"
-    }
-}
-
-single_project_branch {
-    title = "single project (@testGitCommit@)"
-    versions = ["@branchWrapper@"]
-    tasks = [":server:precommit"]
-    gradle-args = ["--no-scan", "--stacktrace"]
-    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "@testGitCommit@"
-    }
-}

build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios

@@ -1,38 +0,0 @@
-// ensure branch scenario is listed first as this is the gradle version that will picked for inspecting the build
-default-scenarios = ["precommit_branch", "precommit_main"]
-
-precommit_main {
-    title = "precommit (main)"
-    versions = ["@mainWrapper@"]
-    cleanup-tasks = ["clean"]
-    tasks = ["precommit"]
-    gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "main"
-    }
-}
-
-precommit_branch {
-    title = "precommit (@testGitCommit@)"
-    versions = ["@branchWrapper@"]
-    cleanup-tasks = ["clean"]
-    tasks = ["precommit"]
-    gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"]
-    run-using = cli // value can be "cli" or "tooling-api"
-    daemon = warm // value can be "warm", "cold", or "none"
-    warm-ups = 5
-    iterations = 10
-    system-properties {
-        "BUILD_PERFORMANCE_TEST" = "true"
-    }
-    git-checkout = {
-        build = "@testGitCommit@"
-    }
-}

build-tools-internal/performance/elasticsearch-esql-tests.scenarios

@@ -0,0 +1,118 @@
+default-scenarios = ["TEST_ESQL_ABI_CHANGE_TAPI", TEST_ESQL_ABI_CHANGE_TAPI_MAIN, "TEST_ESQL_ABI_CHANGE_TAPI_CC", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT", "TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN"]
+
+TEST_ESQL_ABI_CHANGE_TAPI {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["-I", ".ci/init.gradle"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+    git-checkout = {
+        cleanup = "@testGitCommit@"
+        build = "@testGitCommit@"
+    }
+    warm-ups = 3
+    iterations = 10
+}
+
+TEST_ESQL_ABI_CHANGE_TAPI_CC {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+
+    git-checkout = {
+        cleanup = "@testGitCommit@"
+        build = "@testGitCommit@"
+    }
+    warm-ups = 3
+    iterations = 10
+}
+
+TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["--configuration-cache"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+
+    git-checkout = {
+        cleanup = "@testGitCommit@"
+        build = "@testGitCommit@"
+    }
+    warm-ups = 3
+    iterations = 10
+}
+
+TEST_ESQL_ABI_CHANGE_TAPI_MAIN {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["-I", ".ci/init.gradle"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+    git-checkout = {
+        cleanup = "main"
+        build = "main"
+    }
+    warm-ups = 3
+    iterations = 10
+}
+
+TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+
+    git-checkout = {
+        cleanup = "main"
+        build = "main"
+    }
+    warm-ups = 3
+    iterations = 10
+}
+
+TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN {
+    apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
+    tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
+    gradle-args = ["--configuration-cache"]
+    run-using = tooling-api // value can be "cli" or "tooling-api"
+    daemon = warm // value can be "warm", "cold", or "none"
+    copy-file = {
+        source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
+        target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
+        schedule = SCENARIO
+    }
+    git-checkout = {
+        cleanup = "main"
+        build = "main"
+    }
+    warm-ups = 3
+    iterations = 10
+}

build-tools-internal/performance/test-files/BenchmarkingTests.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.gradle.benchmarking;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class BenchmarkingTests extends ESTestCase {
+
+    public void testBenchmarking() {
+        assertTrue(true);
+    }
+}