add CBOR data format support

pom.xml

@@ -232,6 +232,13 @@
             <scope>compile</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-cbor</artifactId>
+            <version>2.3.2</version>
+            <scope>compile</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
@@ -554,6 +561,7 @@
                             <include>com.fasterxml.jackson.core:jackson-core</include>
                             <include>com.fasterxml.jackson.dataformat:jackson-dataformat-smile</include>
                             <include>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</include>
+                            <include>com.fasterxml.jackson.dataformat:jackson-dataformat-cbor</include>
                             <include>joda-time:joda-time</include>
                             <include>org.joda:joda-convert</include>
                             <include>io.netty:netty</include>

src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java

@@ -19,11 +19,13 @@
 
 package org.elasticsearch.common.xcontent;
 
+import com.fasterxml.jackson.dataformat.cbor.CBORConstants;
 import com.fasterxml.jackson.dataformat.smile.SmileConstants;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
@@ -82,6 +84,20 @@ public class XContentFactory {
         return new XContentBuilder(YamlXContent.yamlXContent, os);
     }
 
+    /**
+     * Returns a content builder using CBOR format ({@link org.elasticsearch.common.xcontent.XContentType#CBOR}.
+     */
+    public static XContentBuilder cborBuilder() throws IOException {
+        return contentBuilder(XContentType.CBOR);
+    }
+
+    /**
+     * Constructs a new cbor builder that will output the result into the provided output stream.
+     */
+    public static XContentBuilder cborBuilder(OutputStream os) throws IOException {
+        return new XContentBuilder(CborXContent.cborXContent, os);
+    }
+
     /**
      * Constructs a xcontent builder that will output the result into the provided output stream.
      */
@@ -92,6 +108,8 @@ public class XContentFactory {
             return smileBuilder(outputStream);
         } else if (type == XContentType.YAML) {
             return yamlBuilder(outputStream);
+        } else if (type == XContentType.CBOR) {
+            return cborBuilder(outputStream);
         }
         throw new ElasticsearchIllegalArgumentException("No matching content type for " + type);
     }
@@ -106,6 +124,8 @@ public class XContentFactory {
             return SmileXContent.contentBuilder();
         } else if (type == XContentType.YAML) {
             return YamlXContent.contentBuilder();
+        } else if (type == XContentType.CBOR) {
+            return CborXContent.contentBuilder();
         }
         throw new ElasticsearchIllegalArgumentException("No matching content type for " + type);
     }
@@ -137,6 +157,8 @@ public class XContentFactory {
             return XContentType.YAML;
         }
 
+        // CBOR is not supported
+
         for (int i = 0; i < length; i++) {
             char c = content.charAt(i);
             if (c == '{') {
@@ -209,6 +231,9 @@ public class XContentFactory {
                 return XContentType.YAML;
             }
         }
+        if (first == (CBORConstants.BYTE_OBJECT_INDEFINITE & 0xff)){
+            return XContentType.CBOR;
+        }
         for (int i = 2; i < GUESS_HEADER_LENGTH; i++) {
             int val = si.read();
             if (val == -1) {
@@ -254,6 +279,9 @@ public class XContentFactory {
         if (length > 2 && first == '-' && bytes.get(1) == '-' && bytes.get(2) == '-') {
             return XContentType.YAML;
         }
+        if (first == CBORConstants.BYTE_OBJECT_INDEFINITE){
+            return XContentType.CBOR;
+        }
         for (int i = 0; i < length; i++) {
             if (bytes.get(i) == '{') {
                 return XContentType.JSON;

src/main/java/org/elasticsearch/common/xcontent/XContentType.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
@@ -84,7 +85,26 @@ public enum XContentType {
         public XContent xContent() {
             return YamlXContent.yamlXContent;
         }
-    };
+    },
+    /**
+     * A CBOR based content type.
+     */
+    CBOR(3) {
+        @Override
+        public String restContentType() {
+            return "application/cbor";
+        }
+
+        @Override
+        public String shortName() {
+            return "cbor";
+        }
+
+        @Override
+        public XContent xContent() {
+            return CborXContent.cborXContent;
+        }
+    },;
 
     public static XContentType fromRestContentType(String contentType) {
         if (contentType == null) {
@@ -102,6 +122,10 @@ public enum XContentType {
             return YAML;
         }
 
+        if ("application/cbor".equals(contentType) || "cbor".equalsIgnoreCase(contentType)) {
+            return CBOR;
+        }
+
         return null;
     }
 

src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.cbor;
+
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.*;
+
+import java.io.*;
+
+/**
+ * A CBOR based content implementation using Jackson.
+ */
+public class CborXContent implements XContent {
+
+    public static XContentBuilder contentBuilder() throws IOException {
+        return XContentBuilder.builder(cborXContent);
+    }
+
+    final static CBORFactory cborFactory;
+    public final static CborXContent cborXContent;
+
+    static {
+        cborFactory = new CBORFactory();
+        cborXContent = new CborXContent();
+    }
+
+    private CborXContent() {
+    }
+
+    @Override
+    public XContentType type() {
+        return XContentType.CBOR;
+    }
+
+    @Override
+    public byte streamSeparator() {
+        throw new ElasticsearchParseException("cbor does not support stream parsing...");
+    }
+
+    @Override
+    public XContentGenerator createGenerator(OutputStream os) throws IOException {
+        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8));
+    }
+
+    @Override
+    public XContentGenerator createGenerator(Writer writer) throws IOException {
+        return new CborXContentGenerator(cborFactory.createGenerator(writer));
+    }
+
+    @Override
+    public XContentParser createParser(String content) throws IOException {
+        return new CborXContentParser(cborFactory.createParser(new FastStringReader(content)));
+    }
+
+    @Override
+    public XContentParser createParser(InputStream is) throws IOException {
+        return new CborXContentParser(cborFactory.createParser(is));
+    }
+
+    @Override
+    public XContentParser createParser(byte[] data) throws IOException {
+        return new CborXContentParser(cborFactory.createParser(data));
+    }
+
+    @Override
+    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
+        return new CborXContentParser(cborFactory.createParser(data, offset, length));
+    }
+
+    @Override
+    public XContentParser createParser(BytesReference bytes) throws IOException {
+        if (bytes.hasArray()) {
+            return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
+        }
+        return createParser(bytes.streamInput());
+    }
+
+    @Override
+    public XContentParser createParser(Reader reader) throws IOException {
+        return new CborXContentParser(cborFactory.createParser(reader));
+    }
+}

src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.cbor;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.dataformat.cbor.CBORParser;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ *
+ */
+public class CborXContentGenerator extends JsonXContentGenerator {
+
+    public CborXContentGenerator(JsonGenerator generator) {
+        super(generator);
+    }
+
+    @Override
+    public XContentType contentType() {
+        return XContentType.CBOR;
+    }
+
+    @Override
+    public void usePrintLineFeedAtEnd() {
+        // nothing here
+    }
+
+    @Override
+    public void writeRawField(String fieldName, InputStream content, OutputStream bos) throws IOException {
+        writeFieldName(fieldName);
+        try (CBORParser parser = CborXContent.cborFactory.createParser(content)) {
+            parser.nextToken();
+            generator.copyCurrentStructure(parser);
+        }
+    }
+
+    @Override
+    public void writeRawField(String fieldName, byte[] content, OutputStream bos) throws IOException {
+        writeFieldName(fieldName);
+        try (CBORParser parser = CborXContent.cborFactory.createParser(content)) {
+            parser.nextToken();
+            generator.copyCurrentStructure(parser);
+        }
+    }
+
+    @Override
+    protected void writeObjectRaw(String fieldName, BytesReference content, OutputStream bos) throws IOException {
+        writeFieldName(fieldName);
+        CBORParser parser;
+        if (content.hasArray()) {
+            parser = CborXContent.cborFactory.createParser(content.array(), content.arrayOffset(), content.length());
+        } else {
+            parser = CborXContent.cborFactory.createParser(content.streamInput());
+        }
+        try {
+            parser.nextToken();
+            generator.copyCurrentStructure(parser);
+        } finally {
+            parser.close();
+        }
+    }
+
+    @Override
+    public void writeRawField(String fieldName, byte[] content, int offset, int length, OutputStream bos) throws IOException {
+        writeFieldName(fieldName);
+        try (CBORParser parser = CborXContent.cborFactory.createParser(content, offset, length)) {
+            parser.nextToken();
+            generator.copyCurrentStructure(parser);
+        }
+    }
+}

src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.cbor;
+
+import com.fasterxml.jackson.core.JsonParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContentParser;
+
+/**
+ *
+ */
+public class CborXContentParser extends JsonXContentParser {
+
+    public CborXContentParser(JsonParser parser) {
+        super(parser);
+    }
+
+    @Override
+    public XContentType contentType() {
+        return XContentType.CBOR;
+    }
+}

src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamInput;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ *
+ */
+public class XContentFactoryTests extends ElasticsearchTestCase {
+
+
+    @Test
+    public void testGuessJson() throws IOException {
+        testGuessType(XContentType.JSON);
+    }
+
+    @Test
+    public void testGuessSmile() throws IOException {
+        testGuessType(XContentType.SMILE);
+    }
+
+    @Test
+    public void testGuessYaml() throws IOException {
+        testGuessType(XContentType.YAML);
+    }
+
+    @Test
+    public void testGuessCbor() throws IOException {
+        testGuessType(XContentType.CBOR);
+    }
+
+    private void testGuessType(XContentType type) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(type);
+        builder.startObject();
+        builder.field("field1", "value1");
+        builder.endObject();
+
+        assertThat(XContentFactory.xContentType(builder.bytes()), equalTo(type));
+        BytesArray bytesArray = builder.bytes().toBytesArray();
+        assertThat(XContentFactory.xContentType(new BytesStreamInput(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), false)), equalTo(type));
+
+        // CBOR is binary, cannot use String
+        if (type != XContentType.CBOR) {
+            assertThat(XContentFactory.xContentType(builder.string()), equalTo(type));
+        }
+    }
+}

src/test/java/org/elasticsearch/common/xcontent/builder/BuilderRawFieldTests.java

@@ -52,6 +52,11 @@ public class BuilderRawFieldTests extends ElasticsearchTestCase {
         testRawField(XContentType.YAML);
     }
 
+    @Test
+    public void testCborRawField() throws IOException {
+        testRawField(XContentType.CBOR);
+    }
+
     private void testRawField(XContentType type) throws IOException {
         XContentBuilder builder = XContentFactory.contentBuilder(type);
         builder.startObject();

src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.cbor;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentGenerator;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ *
+ */
+public class JsonVsCborTests extends ElasticsearchTestCase {
+
+    @Test
+    public void compareParsingTokens() throws IOException {
+        BytesStreamOutput xsonOs = new BytesStreamOutput();
+        XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs);
+
+        BytesStreamOutput jsonOs = new BytesStreamOutput();
+        XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs);
+
+        xsonGen.writeStartObject();
+        jsonGen.writeStartObject();
+
+        xsonGen.writeStringField("test", "value");
+        jsonGen.writeStringField("test", "value");
+
+        xsonGen.writeArrayFieldStart("arr");
+        jsonGen.writeArrayFieldStart("arr");
+        xsonGen.writeNumber(1);
+        jsonGen.writeNumber(1);
+        xsonGen.writeNull();
+        jsonGen.writeNull();
+        xsonGen.writeEndArray();
+        jsonGen.writeEndArray();
+
+        xsonGen.writeEndObject();
+        jsonGen.writeEndObject();
+
+        xsonGen.close();
+        jsonGen.close();
+
+        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes().toBytes()));
+    }
+
+    private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException {
+        while (true) {
+            XContentParser.Token token1 = parser1.nextToken();
+            XContentParser.Token token2 = parser2.nextToken();
+            if (token1 == null) {
+                assertThat(token2, nullValue());
+                return;
+            }
+            assertThat(token1, equalTo(token2));
+            switch (token1) {
+                case FIELD_NAME:
+                    assertThat(parser1.currentName(), equalTo(parser2.currentName()));
+                    break;
+                case VALUE_STRING:
+                    assertThat(parser1.text(), equalTo(parser2.text()));
+                    break;
+                case VALUE_NUMBER:
+                    assertThat(parser1.numberType(), equalTo(parser2.numberType()));
+                    assertThat(parser1.numberValue(), equalTo(parser2.numberValue()));
+                    break;
+            }
+        }
+    }
+}