|
|
@@ -72,6 +72,7 @@ import java.util.regex.Pattern;
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
import static java.util.Map.entry;
|
|
|
+import static org.elasticsearch.test.ESTestCase.fail;
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase.constructorWithFunctionInfo;
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase.definition;
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase.functionRegistered;
|
|
|
@@ -79,6 +80,8 @@ import static org.elasticsearch.xpack.esql.expression.function.AbstractFunctionT
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.mapParam;
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.param;
|
|
|
import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.paramWithoutAnnotation;
|
|
|
+import static org.junit.Assert.assertEquals;
|
|
|
+import static org.junit.Assert.assertTrue;
|
|
|
|
|
|
/**
|
|
|
* This class exists to support the new Docs V3 system.
|
|
|
@@ -96,31 +99,32 @@ import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegis
|
|
|
* and partially re-written to satisfy the above requirements.
|
|
|
*/
|
|
|
public abstract class DocsV3Support {
|
|
|
+ private static final Logger logger = LogManager.getLogger(DocsV3Support.class);
|
|
|
|
|
|
private static final String DOCS_WARNING_JSON =
|
|
|
"This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.";
|
|
|
|
|
|
protected static final String DOCS_WARNING = "% " + DOCS_WARNING_JSON + "\n\n";
|
|
|
|
|
|
- static FunctionDocsSupport forFunctions(String name, Class<?> testClass) {
|
|
|
- return new FunctionDocsSupport(name, testClass);
|
|
|
+ static FunctionDocsSupport forFunctions(String name, Class<?> testClass, Callbacks callbacks) {
|
|
|
+ return new FunctionDocsSupport(name, testClass, callbacks);
|
|
|
}
|
|
|
|
|
|
- static OperatorsDocsSupport forOperators(String name, Class<?> testClass) {
|
|
|
- return new OperatorsDocsSupport(name, testClass);
|
|
|
+ static OperatorsDocsSupport forOperators(String name, Class<?> testClass, Callbacks callbacks) {
|
|
|
+ return new OperatorsDocsSupport(name, testClass, callbacks);
|
|
|
}
|
|
|
|
|
|
static void renderDocs(String name, Class<?> testClass) throws Exception {
|
|
|
if (OPERATORS.containsKey(name)) {
|
|
|
- var docs = DocsV3Support.forOperators(name, testClass);
|
|
|
+ var docs = DocsV3Support.forOperators(name, testClass, callbacksFromSystemProperty());
|
|
|
docs.renderSignature();
|
|
|
docs.renderDocs();
|
|
|
} else if (functionRegistered(name)) {
|
|
|
- var docs = DocsV3Support.forFunctions(name, testClass);
|
|
|
+ var docs = DocsV3Support.forFunctions(name, testClass, callbacksFromSystemProperty());
|
|
|
docs.renderSignature();
|
|
|
docs.renderDocs();
|
|
|
} else {
|
|
|
- LogManager.getLogger(testClass).info("Skipping rendering docs because the function '" + name + "' isn't registered");
|
|
|
+ logger.info("Skipping rendering docs because the function '" + name + "' isn't registered");
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -128,9 +132,10 @@ public abstract class DocsV3Support {
|
|
|
@NotNull Constructor<?> ctor,
|
|
|
String name,
|
|
|
Function<String, String> description,
|
|
|
- Class<?> testClass
|
|
|
+ Class<?> testClass,
|
|
|
+ Callbacks callbacks
|
|
|
) throws Exception {
|
|
|
- var docs = forOperators("not " + name.toLowerCase(Locale.ROOT), testClass);
|
|
|
+ var docs = forOperators("not " + name.toLowerCase(Locale.ROOT), testClass, callbacks);
|
|
|
docs.renderDocsForNegatedOperators(ctor, description);
|
|
|
}
|
|
|
|
|
|
@@ -260,19 +265,75 @@ public abstract class DocsV3Support {
|
|
|
return entry(name, new OperatorConfig(name, symbol, clazz, category));
|
|
|
}
|
|
|
|
|
|
- @FunctionalInterface
|
|
|
- interface TempFileWriter {
|
|
|
- void writeToTempDir(Path dir, String extension, String str) throws IOException;
|
|
|
+ public interface Callbacks {
|
|
|
+ void write(Path dir, String name, String extension, String str, boolean kibana) throws IOException;
|
|
|
+
|
|
|
+ boolean supportsRendering();
|
|
|
}
|
|
|
|
|
|
- private class DocsFileWriter implements TempFileWriter {
|
|
|
+ public static class WriteCallbacks implements Callbacks {
|
|
|
@Override
|
|
|
- public void writeToTempDir(Path dir, String extension, String str) throws IOException {
|
|
|
+ public void write(Path dir, String name, String extension, String str, boolean kibana) throws IOException {
|
|
|
Files.createDirectories(dir);
|
|
|
Path file = dir.resolve(name + "." + extension);
|
|
|
Files.writeString(file, str);
|
|
|
logger.info("Wrote to file: {}", file);
|
|
|
}
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean supportsRendering() {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public static class AssertCallbacks implements Callbacks {
|
|
|
+ @Override
|
|
|
+ public void write(Path dir, String name, String extension, String str, boolean kibana) throws IOException {
|
|
|
+ /*
|
|
|
+ * If you've arrived in this method because a CI build is failing and
|
|
|
+ * rerunning the tests doesn't fix it then there is some bug with the
|
|
|
+ * docs generation assertion logic. You can run your REPRODUCE WITH
|
|
|
+ * line prefixed with `BUILDKITE_BUILD_URL=true`.
|
|
|
+ */
|
|
|
+ Path file = dir.resolve(name + "." + extension);
|
|
|
+ assertTrue("rerun test for " + name + ". " + file + " is missing", Files.exists(file));
|
|
|
+ List<String> found = Files.readAllLines(file);
|
|
|
+ List<String> renderedLines = str.lines().toList();
|
|
|
+ int length = Math.min(found.size(), renderedLines.size());
|
|
|
+ for (int i = 0; i < length; i++) {
|
|
|
+ String r = renderedLines.get(i);
|
|
|
+ if (kibana) {
|
|
|
+ r = r.replaceAll("]\\(/reference/([^)\\s]+)\\.md(#\\S+)?\\)", "](https://www.elastic.co/docs/reference/$1$2)");
|
|
|
+ }
|
|
|
+ String f = found.get(i);
|
|
|
+ if (r.equals(f) == false) {
|
|
|
+ assertEquals(errorMessage(name, file, i) + ": mismatch", r, f);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (renderedLines.size() > found.size()) {
|
|
|
+ fail(errorMessage(name, file, found.size()) + ": rendered extra line: " + renderedLines.get(found.size()));
|
|
|
+ }
|
|
|
+ if (renderedLines.size() < found.size()) {
|
|
|
+ fail(errorMessage(name, file, found.size()) + ": found extra line: " + found.get(renderedLines.size()));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private String errorMessage(String name, Path file, int i) {
|
|
|
+ return "rerun tests for " + name + ". Generated docs out of date in " + file + ":" + (i + 1);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean supportsRendering() {
|
|
|
+ /*
|
|
|
+ * Our svg rendering stuff needs the OS it's running on to have
|
|
|
+ * some gui support. Why?! Svg is just xml. And we should be fine
|
|
|
+ * making that. And we are! But we also need to get the sizing
|
|
|
+ * of some fonts and *that* has to render the word. And *that*
|
|
|
+ * needs the gui. Which is wild. And we shouldn't need it. Svg
|
|
|
+ * is a vector format. But that is a problem for another time.
|
|
|
+ */
|
|
|
+ return false;
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -311,13 +372,18 @@ public abstract class DocsV3Support {
|
|
|
protected final String category;
|
|
|
protected final String name;
|
|
|
protected final FunctionDefinition definition;
|
|
|
- protected final Logger logger;
|
|
|
protected final Supplier<Map<List<DataType>, DataType>> signatures;
|
|
|
- private TempFileWriter tempFileWriter;
|
|
|
+ protected final Callbacks callbacks;
|
|
|
private final LicenseRequirementChecker licenseChecker;
|
|
|
|
|
|
- protected DocsV3Support(String category, String name, Class<?> testClass, Supplier<Map<List<DataType>, DataType>> signatures) {
|
|
|
- this(category, name, null, testClass, signatures);
|
|
|
+ protected DocsV3Support(
|
|
|
+ String category,
|
|
|
+ String name,
|
|
|
+ Class<?> testClass,
|
|
|
+ Supplier<Map<List<DataType>, DataType>> signatures,
|
|
|
+ Callbacks callbacks
|
|
|
+ ) {
|
|
|
+ this(category, name, null, testClass, signatures, callbacks);
|
|
|
}
|
|
|
|
|
|
private DocsV3Support(
|
|
|
@@ -325,22 +391,17 @@ public abstract class DocsV3Support {
|
|
|
String name,
|
|
|
FunctionDefinition definition,
|
|
|
Class<?> testClass,
|
|
|
- Supplier<Map<List<DataType>, DataType>> signatures
|
|
|
+ Supplier<Map<List<DataType>, DataType>> signatures,
|
|
|
+ Callbacks callbacks
|
|
|
) {
|
|
|
this.category = category;
|
|
|
this.name = name;
|
|
|
this.definition = definition == null ? definition(name) : definition;
|
|
|
- this.logger = LogManager.getLogger(testClass);
|
|
|
this.signatures = signatures;
|
|
|
- this.tempFileWriter = new DocsFileWriter();
|
|
|
+ this.callbacks = callbacks;
|
|
|
this.licenseChecker = new LicenseRequirementChecker(testClass);
|
|
|
}
|
|
|
|
|
|
- /** Used in tests to capture output for asserting on the content */
|
|
|
- void setTempFileWriter(TempFileWriter tempFileWriter) {
|
|
|
- this.tempFileWriter = tempFileWriter;
|
|
|
- }
|
|
|
-
|
|
|
String replaceLinks(String text) {
|
|
|
return replaceAsciidocLinks(replaceMacros(text));
|
|
|
}
|
|
|
@@ -475,9 +536,10 @@ public abstract class DocsV3Support {
|
|
|
}
|
|
|
|
|
|
void writeToTempImageDir(String str) throws IOException {
|
|
|
+ assert callbacks.supportsRendering();
|
|
|
// We have to write to a tempdir because it’s all test are allowed to write to. Gradle can move them.
|
|
|
Path dir = PathUtils.get(System.getProperty("java.io.tmpdir")).resolve("esql").resolve("images").resolve(category);
|
|
|
- tempFileWriter.writeToTempDir(dir, "svg", str);
|
|
|
+ callbacks.write(dir, name, "svg", str, false);
|
|
|
}
|
|
|
|
|
|
void writeToTempSnippetsDir(String subdir, String str) throws IOException {
|
|
|
@@ -487,13 +549,13 @@ public abstract class DocsV3Support {
|
|
|
.resolve("_snippets")
|
|
|
.resolve(category)
|
|
|
.resolve(subdir);
|
|
|
- tempFileWriter.writeToTempDir(dir, "md", str);
|
|
|
+ callbacks.write(dir, name, "md", str, false);
|
|
|
}
|
|
|
|
|
|
void writeToTempKibanaDir(String subdir, String extension, String str) throws IOException {
|
|
|
// We have to write to a tempdir because it’s all test are allowed to write to. Gradle can move them.
|
|
|
Path dir = PathUtils.get(System.getProperty("java.io.tmpdir")).resolve("esql").resolve("kibana").resolve(subdir).resolve(category);
|
|
|
- tempFileWriter.writeToTempDir(dir, extension, str);
|
|
|
+ callbacks.write(dir, name, extension, str, true);
|
|
|
}
|
|
|
|
|
|
protected abstract void renderSignature() throws IOException;
|
|
|
@@ -501,21 +563,25 @@ public abstract class DocsV3Support {
|
|
|
protected abstract void renderDocs() throws Exception;
|
|
|
|
|
|
static class FunctionDocsSupport extends DocsV3Support {
|
|
|
- private FunctionDocsSupport(String name, Class<?> testClass) {
|
|
|
- super("functions", name, testClass, () -> AbstractFunctionTestCase.signatures(testClass));
|
|
|
+ private FunctionDocsSupport(String name, Class<?> testClass, Callbacks callbacks) {
|
|
|
+ super("functions", name, testClass, () -> AbstractFunctionTestCase.signatures(testClass), callbacks);
|
|
|
}
|
|
|
|
|
|
FunctionDocsSupport(
|
|
|
String name,
|
|
|
Class<?> testClass,
|
|
|
FunctionDefinition definition,
|
|
|
- Supplier<Map<List<DataType>, DataType>> signatures
|
|
|
+ Supplier<Map<List<DataType>, DataType>> signatures,
|
|
|
+ Callbacks callbacks
|
|
|
) {
|
|
|
- super("functions", name, definition, testClass, signatures);
|
|
|
+ super("functions", name, definition, testClass, signatures, callbacks);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
protected void renderSignature() throws IOException {
|
|
|
+ if (callbacks.supportsRendering() == false) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
String rendered = buildFunctionSignatureSvg();
|
|
|
if (rendered == null) {
|
|
|
logger.info("Skipping rendering signature because the function '{}' isn't registered", name);
|
|
|
@@ -681,22 +747,26 @@ public abstract class DocsV3Support {
|
|
|
public static class OperatorsDocsSupport extends DocsV3Support {
|
|
|
private final OperatorConfig op;
|
|
|
|
|
|
- private OperatorsDocsSupport(String name, Class<?> testClass) {
|
|
|
- this(name, testClass, OPERATORS.get(name), () -> AbstractFunctionTestCase.signatures(testClass));
|
|
|
+ private OperatorsDocsSupport(String name, Class<?> testClass, Callbacks callbacks) {
|
|
|
+ this(name, testClass, OPERATORS.get(name), () -> AbstractFunctionTestCase.signatures(testClass), callbacks);
|
|
|
}
|
|
|
|
|
|
public OperatorsDocsSupport(
|
|
|
String name,
|
|
|
Class<?> testClass,
|
|
|
OperatorConfig op,
|
|
|
- Supplier<Map<List<DataType>, DataType>> signatures
|
|
|
+ Supplier<Map<List<DataType>, DataType>> signatures,
|
|
|
+ Callbacks callbacks
|
|
|
) {
|
|
|
- super("operators", name, testClass, signatures);
|
|
|
+ super("operators", name, testClass, signatures, callbacks);
|
|
|
this.op = op;
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
public void renderSignature() throws IOException {
|
|
|
+ if (callbacks.supportsRendering() == false) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
String rendered = (switch (op.category()) {
|
|
|
case BINARY -> RailRoadDiagram.infixOperator("lhs", op.symbol(), "rhs");
|
|
|
case UNARY -> RailRoadDiagram.prefixOperator(op.symbol(), "v");
|
|
|
@@ -871,9 +941,10 @@ public abstract class DocsV3Support {
|
|
|
Class<?> testClass,
|
|
|
LogicalPlan command,
|
|
|
XPackLicenseState licenseState,
|
|
|
- ObservabilityTier observabilityTier
|
|
|
+ ObservabilityTier observabilityTier,
|
|
|
+ Callbacks callbacks
|
|
|
) {
|
|
|
- super("commands", name, testClass, Map::of);
|
|
|
+ super("commands", name, testClass, Map::of, callbacks);
|
|
|
this.command = command;
|
|
|
this.licenseState = licenseState;
|
|
|
this.observabilityTier = observabilityTier;
|
|
|
@@ -885,9 +956,10 @@ public abstract class DocsV3Support {
|
|
|
Class<?> testClass,
|
|
|
LogicalPlan command,
|
|
|
List<EsqlFunctionRegistry.ArgSignature> args,
|
|
|
- Supplier<Map<List<DataType>, DataType>> signatures
|
|
|
+ Supplier<Map<List<DataType>, DataType>> signatures,
|
|
|
+ Callbacks callbacks
|
|
|
) {
|
|
|
- super("commands", name, testClass, signatures);
|
|
|
+ super("commands", name, testClass, signatures, callbacks);
|
|
|
this.command = command;
|
|
|
this.args = args;
|
|
|
this.licenseState = null;
|
|
|
@@ -1422,4 +1494,13 @@ public abstract class DocsV3Support {
|
|
|
}
|
|
|
return sb.append(" |\n").toString();
|
|
|
}
|
|
|
+
|
|
|
+ public static Callbacks callbacksFromSystemProperty() {
|
|
|
+ String prop = System.getProperty("generateDocs");
|
|
|
+ return switch (prop) {
|
|
|
+ case "write" -> new WriteCallbacks();
|
|
|
+ case "assert" -> new AssertCallbacks();
|
|
|
+ default -> throw new IllegalArgumentException("unsupported value for generateDocs [" + prop + "]");
|
|
|
+ };
|
|
|
+ }
|
|
|
}
|
Nik Everett