Merge remote-tracking branch 'dakrone/allow-bad-json'

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -46,11 +46,30 @@ public class JsonXContent implements XContent {
     }
 
     private final static JsonFactory jsonFactory;
+    public final static String JSON_ALLOW_UNQUOTED_FIELD_NAMES = "elasticsearch.json.allow_unquoted_field_names";
     public final static JsonXContent jsonXContent;
+    public final static boolean unquotedFieldNamesSet;
 
     static {
         jsonFactory = new JsonFactory();
-        jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+        // TODO: Remove the system property configuration for this in Elasticsearch 6.0.0
+        String jsonUnquoteProp = System.getProperty(JSON_ALLOW_UNQUOTED_FIELD_NAMES);
+        if (jsonUnquoteProp == null) {
+            unquotedFieldNamesSet = false;
+            jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+        } else {
+            unquotedFieldNamesSet = true;
+            switch (jsonUnquoteProp) {
+                case "true":
+                    jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+                    break;
+                case "false":
+                    jsonFactory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, false);
+                    break;
+                default:
+                    throw new IllegalArgumentException("invalid value for [" + JSON_ALLOW_UNQUOTED_FIELD_NAMES + "]: " + jsonUnquoteProp);
+            }
+        }
         jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
         jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
         jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...

core/src/main/java/org/elasticsearch/node/Node.java

@@ -47,6 +47,7 @@ import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -60,6 +61,7 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -174,6 +176,12 @@ public class Node implements Closeable {
             logger.debug("using config [{}], data [{}], logs [{}], plugins [{}]",
                     tmpEnv.configFile(), Arrays.toString(tmpEnv.dataFiles()), tmpEnv.logsFile(), tmpEnv.pluginsFile());
         }
+        // TODO: Remove this in Elasticsearch 6.0.0
+        if (JsonXContent.unquotedFieldNamesSet) {
+            DeprecationLogger dLogger = new DeprecationLogger(logger);
+            dLogger.deprecated("[{}] has been set, but will be removed in Elasticsearch 6.0.0",
+                    JsonXContent.JSON_ALLOW_UNQUOTED_FIELD_NAMES);
+        }
 
         this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins);
         this.settings = pluginsService.updatedSettings();

distribution/src/main/resources/config/jvm.options

@@ -79,3 +79,11 @@
 # log GC status to a file with time stamps
 # ensure the directory exists
 #-Xloggc:${loggc}
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true

docs/reference/migration/migrate_5_0/rest.asciidoc

@@ -38,5 +38,12 @@ empty, that means that the node is a coordinating only node.
 
 ==== Forbid unquoted JSON
 
-Previously, JSON documents were allowed with unquoted field names, which
-isn't strictly JSON and broke some Elasticsearch clients.
+Previously, JSON documents were allowed with unquoted field names, which isn't
+strictly JSON and broke some Elasticsearch clients. If documents were already
+indexed with unquoted fields in a previous vesrion of Elasticsearch, some
+operations may throw errors. To accompany this, a commented out JVM option has
+been added to the `jvm.options` file:
+`-Delasticsearch.json.allow_unquoted_field_names`.
+
+Note that this option is provided solely for migration purposes and will be
+removed in Elasticsearch 6.0.0.

qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats

@@ -115,6 +115,24 @@ setup() {
     export ES_JAVA_OPTS=$es_java_opts
 }
 
+@test "[TAR]" start Elasticsearch with unquoted JSON option {
+    local es_java_opts=$ES_JAVA_OPTS
+    local es_jvm_options=$ES_JVM_OPTIONS
+    local temp=`mktemp -d`
+    touch "$temp/jvm.options"
+    chown -R elasticsearch:elasticsearch "$temp"
+    echo "-Delasticsearch.json.allow_unquoted_field_names=true" >> "$temp/jvm.options"
+    export ES_JVM_OPTIONS="$temp/jvm.options"
+    start_elasticsearch_service
+    # unquoted field name
+    curl -s -XPOST localhost:9200/i/d/1 -d'{foo: "bar"}'
+    [ "$?" -eq 0 ]
+    curl -s -XDELETE localhost:9200/i
+    stop_elasticsearch_service
+    export ES_JVM_OPTIONS=$es_jvm_options
+    export ES_JAVA_OPTS=$es_java_opts
+}
+
 @test "[TAR]" remove tar {
     rm -rf "/tmp/elasticsearch"
 }