|
|
@@ -1,585 +0,0 @@
|
|
|
-# Licensed to Elasticsearch under one or more contributor
|
|
|
-# license agreements. See the NOTICE file distributed with
|
|
|
-# this work for additional information regarding copyright
|
|
|
-# ownership. Elasticsearch licenses this file to you under
|
|
|
-# the Apache License, Version 2.0 (the "License"); you may
|
|
|
-# not use this file except in compliance with the License.
|
|
|
-# You may obtain a copy of the License at
|
|
|
-#
|
|
|
-# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
-#
|
|
|
-# Unless required by applicable law or agreed to in writing,
|
|
|
-# software distributed under the License is distributed on
|
|
|
-# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
|
|
-# either express or implied. See the License for the specific
|
|
|
-# language governing permissions and limitations under the License.
|
|
|
-
|
|
|
-import argparse
|
|
|
-import base64
|
|
|
-import glob
|
|
|
-import logging
|
|
|
-import os
|
|
|
-import random
|
|
|
-import shutil
|
|
|
-import subprocess
|
|
|
-import sys
|
|
|
-import tempfile
|
|
|
-import time
|
|
|
-
|
|
|
-DEFAULT_TRANSPORT_TCP_PORT = 9300
|
|
|
-DEFAULT_HTTP_TCP_PORT = 9200
|
|
|
-
|
|
|
-if sys.version_info[0] < 3:
|
|
|
- print('%s must use python 3.x (for the ES python client)' % sys.argv[0])
|
|
|
-
|
|
|
-try:
|
|
|
- from elasticsearch import Elasticsearch
|
|
|
- from elasticsearch.exceptions import ConnectionError
|
|
|
- from elasticsearch.exceptions import TransportError
|
|
|
-except ImportError as e:
|
|
|
- print('Can\'t import elasticsearch please install `sudo pip3 install elasticsearch`')
|
|
|
- sys.exit(1)
|
|
|
-
|
|
|
-# sometimes returns True
|
|
|
-def rarely():
|
|
|
- return random.randint(0, 10) == 0
|
|
|
-
|
|
|
-# usually returns True
|
|
|
-def frequently():
|
|
|
- return not rarely()
|
|
|
-
|
|
|
-def capabilities_of(version):
|
|
|
- current_version = parse_version(version)
|
|
|
-
|
|
|
- return {
|
|
|
- 'warmers': current_version < parse_version('2.0.0-alpha1'),
|
|
|
- 'dots_in_field_names': current_version >= parse_version('2.4.0'),
|
|
|
- 'lenient_booleans': current_version < parse_version('6.0.0-alpha1')
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def falsy(lenient):
|
|
|
- return random.choice(['off', 'no', '0', 0, 'false', False]) if lenient else False
|
|
|
-
|
|
|
-
|
|
|
-def truthy(lenient):
|
|
|
- return random.choice(['on', 'yes', '1', 1, 'true', True]) if lenient else True
|
|
|
-
|
|
|
-
|
|
|
-def random_bool(lenient):
|
|
|
- return random.choice([falsy, truthy])(lenient)
|
|
|
-
|
|
|
-
|
|
|
-# asserts the correctness of the given hits given they are sorted asc
|
|
|
-def assert_sort(hits):
|
|
|
- values = [hit['sort'] for hit in hits['hits']['hits']]
|
|
|
- assert len(values) > 0, 'expected non emtpy result'
|
|
|
- val = min(values)
|
|
|
- for x in values:
|
|
|
- assert x >= val, '%s >= %s' % (x, val)
|
|
|
- val = x
|
|
|
-
|
|
|
-# Indexes the given number of document into the given index
|
|
|
-# and randomly runs refresh, optimize and flush commands
|
|
|
-def index_documents(es, index_name, type, num_docs, capabilities):
|
|
|
- logging.info('Indexing %s docs' % num_docs)
|
|
|
- index(es, index_name, type, num_docs, capabilities, flush=True)
|
|
|
- logging.info('Flushing index')
|
|
|
- es.indices.flush(index=index_name)
|
|
|
-
|
|
|
-def index(es, index_name, type, num_docs, capabilities, flush=False):
|
|
|
- for id in range(0, num_docs):
|
|
|
- lenient_bool = capabilities['lenient_booleans']
|
|
|
- body = {
|
|
|
- 'string': str(random.randint(0, 100)),
|
|
|
- 'long_sort': random.randint(0, 100),
|
|
|
- 'double_sort': float(random.randint(0, 100)),
|
|
|
- # be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
|
|
|
- 'bool': random_bool(lenient_bool) if id > 0 else random.choice([True, False])
|
|
|
- }
|
|
|
- if capabilities['dots_in_field_names']:
|
|
|
- body['field.with.dots'] = str(random.randint(0, 100))
|
|
|
-
|
|
|
- body['binary'] = base64.b64encode(bytearray(random.getrandbits(8) for _ in range(16))).decode('ascii')
|
|
|
-
|
|
|
- es.index(index=index_name, doc_type=type, id=id, body=body)
|
|
|
-
|
|
|
- if rarely():
|
|
|
- es.indices.refresh(index=index_name)
|
|
|
- if rarely() and flush:
|
|
|
- es.indices.flush(index=index_name, force=frequently())
|
|
|
-
|
|
|
-def reindex_docs(es, index_name, type, num_docs, capabilities):
|
|
|
- logging.info('Re-indexing %s docs' % num_docs)
|
|
|
- # TODO: Translog recovery fails on mixed representation of booleans as strings / booleans (e.g. "true", true)
|
|
|
- # (see gradle :core:test -Dtests.seed=AF7BB7B3FA387AAE -Dtests.class=org.elasticsearch.index.engine.InternalEngineTests
|
|
|
- # -Dtests.method="testUpgradeOldIndex")
|
|
|
- capabilities['lenient_booleans'] = False
|
|
|
- # reindex some docs after the flush such that we have something in the translog
|
|
|
- index(es, index_name, type, num_docs, capabilities)
|
|
|
-
|
|
|
-def delete_by_query(es, version, index_name, doc_type):
|
|
|
-
|
|
|
- logging.info('Deleting long_sort:[10..20] docs')
|
|
|
-
|
|
|
- query = {'query':
|
|
|
- {'range':
|
|
|
- {'long_sort':
|
|
|
- {'gte': 10,
|
|
|
- 'lte': 20}}}}
|
|
|
-
|
|
|
- if version.startswith('0.') or version in ('1.0.0.Beta1', '1.0.0.Beta2'):
|
|
|
- # TODO #10262: we can't write DBQ into the translog for these old versions until we fix this back-compat bug:
|
|
|
-
|
|
|
- # #4074: these versions don't expect to see the top-level 'query' to count/delete_by_query:
|
|
|
- query = query['query']
|
|
|
- return
|
|
|
-
|
|
|
- deleted_count = es.count(index=index_name, doc_type=doc_type, body=query)['count']
|
|
|
-
|
|
|
- result = es.delete_by_query(index=index_name,
|
|
|
- doc_type=doc_type,
|
|
|
- body=query)
|
|
|
-
|
|
|
- # make sure no shards failed:
|
|
|
- assert result['_indices'][index_name]['_shards']['failed'] == 0, 'delete by query failed: %s' % result
|
|
|
-
|
|
|
- logging.info('Deleted %d docs' % deleted_count)
|
|
|
-
|
|
|
-def run_basic_asserts(es, version, index_name, type, num_docs):
|
|
|
- count = es.count(index=index_name)['count']
|
|
|
- assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count)
|
|
|
- if parse_version(version) < parse_version('5.1.0'):
|
|
|
- # This alias isn't allowed to be created after 5.1 so we can verify that we can still use it
|
|
|
- count = es.count(index='#' + index_name)['count']
|
|
|
- assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count)
|
|
|
- for _ in range(0, num_docs):
|
|
|
- random_doc_id = random.randint(0, num_docs-1)
|
|
|
- doc = es.get(index=index_name, doc_type=type, id=random_doc_id)
|
|
|
- assert doc, 'Expected document for id %s but got %s' % (random_doc_id, doc)
|
|
|
-
|
|
|
- assert_sort(es.search(index=index_name,
|
|
|
- body={
|
|
|
- 'sort': [
|
|
|
- {'double_sort': {'order': 'asc'}}
|
|
|
- ]
|
|
|
- }))
|
|
|
-
|
|
|
- assert_sort(es.search(index=index_name,
|
|
|
- body={
|
|
|
- 'sort': [
|
|
|
- {'long_sort': {'order': 'asc'}}
|
|
|
- ]
|
|
|
- }))
|
|
|
-
|
|
|
-
|
|
|
-def build_version(version_tuple):
|
|
|
- return '.'.join([str(x) for x in version_tuple])
|
|
|
-
|
|
|
-def build_tuple(version_string):
|
|
|
- return [int(x) for x in version_string.split('.')]
|
|
|
-
|
|
|
-def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSPORT_TCP_PORT, http_port=DEFAULT_HTTP_TCP_PORT, cluster_name=None):
|
|
|
- logging.info('Starting node from %s on port %s/%s, data_dir %s' % (release_dir, tcp_port, http_port, data_dir))
|
|
|
- if cluster_name is None:
|
|
|
- cluster_name = 'bwc_index_' + version
|
|
|
- if parse_version(version) < parse_version("5.0.0-alpha1"):
|
|
|
- prefix = '-Des.'
|
|
|
- else:
|
|
|
- prefix = '-E'
|
|
|
- cmd = [
|
|
|
- os.path.join(release_dir, 'bin/elasticsearch'),
|
|
|
- '%spath.data=%s' % (prefix, data_dir),
|
|
|
- '%spath.logs=logs' % prefix,
|
|
|
- '%scluster.name=%s' % (prefix, cluster_name),
|
|
|
- '%snetwork.host=localhost' % prefix,
|
|
|
- '%stransport.tcp.port=%s' % (prefix, tcp_port),
|
|
|
- '%shttp.port=%s' % (prefix, http_port),
|
|
|
- '%spath.repo=%s' % (prefix, repo_dir)
|
|
|
- ]
|
|
|
- if version.startswith('0.') or version.startswith('1.0.0.Beta') :
|
|
|
- cmd.append('-f') # version before 1.0 start in background automatically
|
|
|
- return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
|
- env=dict(os.environ, ES_JAVA_OPTS='-Dmapper.allow_dots_in_name=true'))
|
|
|
-
|
|
|
-def install_plugin(version, release_dir, plugin_name):
|
|
|
- run_plugin(version, release_dir, 'install', [plugin_name])
|
|
|
-
|
|
|
-def remove_plugin(version, release_dir, plugin_name):
|
|
|
- run_plugin(version, release_dir, 'remove', [plugin_name])
|
|
|
-
|
|
|
-def run_plugin(version, release_dir, plugin_cmd, args):
|
|
|
- cmd = [os.path.join(release_dir, 'bin/elasticsearch-plugin'), plugin_cmd] + args
|
|
|
- subprocess.check_call(cmd)
|
|
|
-
|
|
|
-def create_client(http_port=DEFAULT_HTTP_TCP_PORT, timeout=30):
|
|
|
- logging.info('Waiting for node to startup')
|
|
|
- for _ in range(0, timeout):
|
|
|
- # TODO: ask Honza if there is a better way to do this?
|
|
|
- try:
|
|
|
- client = Elasticsearch([{'host': 'localhost', 'port': http_port}])
|
|
|
- client.cluster.health(wait_for_nodes=1)
|
|
|
- client.count() # can we actually search or do we get a 503? -- anyway retry
|
|
|
- return client
|
|
|
- except (ConnectionError, TransportError):
|
|
|
- pass
|
|
|
- time.sleep(1)
|
|
|
- assert False, 'Timed out waiting for node for %s seconds' % timeout
|
|
|
-
|
|
|
-def generate_index(client, version, index_name):
|
|
|
- client.indices.delete(index=index_name, ignore=404)
|
|
|
- logging.info('Create single shard test index')
|
|
|
-
|
|
|
- capabilities = capabilities_of(version)
|
|
|
- lenient_booleans = capabilities['lenient_booleans']
|
|
|
-
|
|
|
- mappings = {}
|
|
|
- warmers = {}
|
|
|
- if capabilities['warmers']:
|
|
|
- warmers['warmer1'] = {
|
|
|
- 'source': {
|
|
|
- 'query': {
|
|
|
- 'match_all': {}
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- # backcompat test for legacy type level analyzer settings, see #8874
|
|
|
- mappings['analyzer_type1'] = {
|
|
|
- 'analyzer': 'standard',
|
|
|
- 'properties': {
|
|
|
- 'string_with_index_analyzer': {
|
|
|
- 'type': 'string',
|
|
|
- 'index_analyzer': 'standard'
|
|
|
- },
|
|
|
- }
|
|
|
- }
|
|
|
- # completion type was added in 0.90.3
|
|
|
- if not version.startswith('0.20') and version not in ['0.90.0.Beta1', '0.90.0.RC1', '0.90.0.RC2', '0.90.0', '0.90.1', '0.90.2']:
|
|
|
- mappings['analyzer_type1']['properties']['completion_with_index_analyzer'] = {
|
|
|
- 'type': 'completion',
|
|
|
- 'index_analyzer': 'standard'
|
|
|
- }
|
|
|
-
|
|
|
- mappings['analyzer_type2'] = {
|
|
|
- 'index_analyzer': 'standard',
|
|
|
- 'search_analyzer': 'keyword',
|
|
|
- 'search_quote_analyzer': 'english',
|
|
|
- }
|
|
|
- mappings['index_name_and_path'] = {
|
|
|
- 'properties': {
|
|
|
- 'parent_multi_field': {
|
|
|
- 'type': 'string',
|
|
|
- 'path': 'just_name',
|
|
|
- 'fields': {
|
|
|
- 'raw': {'type': 'string', 'index': 'not_analyzed', 'index_name': 'raw_multi_field'}
|
|
|
- }
|
|
|
- },
|
|
|
- 'field_with_index_name': {
|
|
|
- 'type': 'string',
|
|
|
- 'index_name': 'custom_index_name_for_field'
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- mappings['meta_fields'] = {
|
|
|
- '_routing': {
|
|
|
- 'required': falsy(lenient_booleans)
|
|
|
- },
|
|
|
- }
|
|
|
- mappings['custom_formats'] = {
|
|
|
- 'properties': {
|
|
|
- 'string_with_custom_postings': {
|
|
|
- 'type': 'string',
|
|
|
- 'postings_format': 'Lucene41'
|
|
|
- },
|
|
|
- 'long_with_custom_doc_values': {
|
|
|
- 'type': 'long',
|
|
|
- 'doc_values_format': 'Lucene42'
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- mappings['auto_boost'] = {
|
|
|
- '_all': {
|
|
|
- 'auto_boost': truthy(lenient_booleans)
|
|
|
- }
|
|
|
- }
|
|
|
- mappings['doc'] = {'properties' : {}}
|
|
|
-
|
|
|
- if capabilities['dots_in_field_names']:
|
|
|
- if parse_version(version) < parse_version("5.0.0-alpha1"):
|
|
|
- mappings["doc"]['properties'].update({
|
|
|
- 'field.with.dots': {
|
|
|
- 'type': 'string',
|
|
|
- 'boost': 4
|
|
|
- }
|
|
|
- })
|
|
|
- else:
|
|
|
- mappings["doc"]['properties'].update({
|
|
|
- 'field.with.dots': {
|
|
|
- 'type': 'text'
|
|
|
- }
|
|
|
- })
|
|
|
-
|
|
|
- if parse_version(version) < parse_version("5.0.0-alpha1"):
|
|
|
- mappings['norms'] = {
|
|
|
- 'properties': {
|
|
|
- 'string_with_norms_disabled': {
|
|
|
- 'type': 'string',
|
|
|
- 'norms' : {
|
|
|
- 'enabled' : False
|
|
|
- }
|
|
|
- },
|
|
|
- 'string_with_norms_enabled': {
|
|
|
- 'type': 'string',
|
|
|
- 'index': 'not_analyzed',
|
|
|
- 'norms': {
|
|
|
- 'enabled' : True,
|
|
|
- 'loading': 'eager'
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- mappings['doc'] = {
|
|
|
- 'properties': {
|
|
|
- 'string': {
|
|
|
- 'type': 'string',
|
|
|
- 'boost': 4
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- else: # current version of the norms mapping
|
|
|
- mappings['norms'] = {
|
|
|
- 'properties': {
|
|
|
- 'string_with_norms_disabled': {
|
|
|
- 'type': 'text',
|
|
|
- 'norms': False
|
|
|
- },
|
|
|
- 'string_with_norms_enabled': {
|
|
|
- 'type': 'keyword',
|
|
|
- 'index': 'not_analyzed',
|
|
|
- 'norms': True,
|
|
|
- 'eager_global_ordinals' : True
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- mappings['doc']['properties'].update({
|
|
|
- 'string': {
|
|
|
- 'type': 'text',
|
|
|
- 'boost': 4
|
|
|
- }
|
|
|
- })
|
|
|
-
|
|
|
- # test back-compat of stored binary fields
|
|
|
- mappings['doc']['properties']['binary'] = {
|
|
|
- 'type': 'binary',
|
|
|
- 'store': truthy(lenient_booleans),
|
|
|
- }
|
|
|
-
|
|
|
- settings = {
|
|
|
- 'number_of_shards': 1,
|
|
|
- 'number_of_replicas': 0,
|
|
|
- }
|
|
|
- if version.startswith('0.') or version.startswith('1.'):
|
|
|
- # Same as ES default (60 seconds), but missing the units to make sure they are inserted on upgrade:
|
|
|
- settings['gc_deletes'] = '60000',
|
|
|
- # Same as ES default (5 GB), but missing the units to make sure they are inserted on upgrade:
|
|
|
- settings['merge.policy.max_merged_segment'] = '5368709120'
|
|
|
- body = {
|
|
|
- 'settings': settings,
|
|
|
- 'mappings': mappings,
|
|
|
- }
|
|
|
-
|
|
|
- if warmers:
|
|
|
- body['warmers'] = warmers
|
|
|
- client.indices.create(index=index_name, body=body)
|
|
|
- if parse_version(version) < parse_version("5.0.0-alpha1"):
|
|
|
- health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0)
|
|
|
- else:
|
|
|
- health = client.cluster.health(wait_for_status='green', wait_for_no_relocating_shards=True)
|
|
|
- assert health['timed_out'] == False, 'cluster health timed out %s' % health
|
|
|
-
|
|
|
- num_docs = random.randint(2000, 3000)
|
|
|
- if version == "1.1.0":
|
|
|
- # 1.1.0 is buggy and creates lots and lots of segments, so we create a
|
|
|
- # lighter index for it to keep bw tests reasonable
|
|
|
- # see https://github.com/elastic/elasticsearch/issues/5817
|
|
|
- num_docs = int(num_docs / 10)
|
|
|
- index_documents(client, index_name, 'doc', num_docs, capabilities)
|
|
|
- if parse_version(version) < parse_version('5.1.0'):
|
|
|
- logging.info("Adding a alias that can't be created in 5.1+ so we can assert that we can still use it")
|
|
|
- client.indices.put_alias(index=index_name, name='#' + index_name)
|
|
|
- logging.info('Running basic asserts on the data added')
|
|
|
- run_basic_asserts(client, version, index_name, 'doc', num_docs)
|
|
|
- return num_docs, capabilities
|
|
|
-
|
|
|
-def snapshot_index(client, version, repo_dir):
|
|
|
- persistent = {
|
|
|
- 'cluster.routing.allocation.exclude.version_attr': version
|
|
|
- }
|
|
|
- if parse_version(version) < parse_version('5.0.0-alpha1'):
|
|
|
- # Same as ES default (30 seconds), but missing the units to make sure they are inserted on upgrade:
|
|
|
- persistent['discovery.zen.publish_timeout'] = '30000'
|
|
|
- # Same as ES default (512 KB), but missing the units to make sure they are inserted on upgrade:
|
|
|
- persistent['indices.recovery.file_chunk_size'] = '524288'
|
|
|
- # Add bogus persistent settings to make sure they can be restored
|
|
|
- client.cluster.put_settings(body={
|
|
|
- 'persistent': persistent
|
|
|
- })
|
|
|
- client.indices.put_template(name='template_' + version.lower(), order=0, body={
|
|
|
- "template": "te*",
|
|
|
- "settings": {
|
|
|
- "number_of_shards" : 1
|
|
|
- },
|
|
|
- "mappings": {
|
|
|
- "type1": {
|
|
|
- "_source": {
|
|
|
- "enabled": falsy(capabilities_of(version)['lenient_booleans'])
|
|
|
- }
|
|
|
- }
|
|
|
- },
|
|
|
- "aliases": {
|
|
|
- "alias1": {},
|
|
|
- "alias2": {
|
|
|
- "filter": {
|
|
|
- "term": {"version" : version }
|
|
|
- },
|
|
|
- "routing": "kimchy"
|
|
|
- },
|
|
|
- "{index}-alias": {}
|
|
|
- }
|
|
|
- })
|
|
|
- client.snapshot.create_repository(repository='test_repo', body={
|
|
|
- 'type': 'fs',
|
|
|
- 'settings': {
|
|
|
- 'location': repo_dir
|
|
|
- }
|
|
|
- })
|
|
|
- client.snapshot.create(repository='test_repo', snapshot='test_1', wait_for_completion=True)
|
|
|
- client.snapshot.delete_repository(repository='test_repo')
|
|
|
-
|
|
|
-def compress_index(version, tmp_dir, output_dir):
|
|
|
- compress(tmp_dir, output_dir, 'index-%s.zip' % version, 'data')
|
|
|
-
|
|
|
-def compress_repo(version, tmp_dir, output_dir):
|
|
|
- compress(tmp_dir, output_dir, 'repo-%s.zip' % version, 'repo')
|
|
|
-
|
|
|
-def compress(tmp_dir, output_dir, zipfile, directory):
|
|
|
- abs_output_dir = os.path.abspath(output_dir)
|
|
|
- zipfile = os.path.join(abs_output_dir, zipfile)
|
|
|
- if os.path.exists(zipfile):
|
|
|
- os.remove(zipfile)
|
|
|
- logging.info('Compressing index into %s, tmpDir %s', zipfile, tmp_dir)
|
|
|
- olddir = os.getcwd()
|
|
|
- os.chdir(tmp_dir)
|
|
|
- subprocess.check_call('zip -r %s %s' % (zipfile, directory), shell=True)
|
|
|
- os.chdir(olddir)
|
|
|
-
|
|
|
-
|
|
|
-def parse_config():
|
|
|
- parser = argparse.ArgumentParser(description='Builds an elasticsearch index for backwards compatibility tests')
|
|
|
- required = parser.add_mutually_exclusive_group(required=True)
|
|
|
- required.add_argument('versions', metavar='X.Y.Z', nargs='*', default=[],
|
|
|
- help='The elasticsearch version to build an index for')
|
|
|
- required.add_argument('--all', action='store_true', default=False,
|
|
|
- help='Recreate all existing backwards compatibility indexes')
|
|
|
- parser.add_argument('--releases-dir', '-d', default='backwards', metavar='DIR',
|
|
|
- help='The directory containing elasticsearch releases')
|
|
|
- parser.add_argument('--output-dir', '-o', default='core/src/test/resources/indices/bwc',
|
|
|
- help='The directory to write the zipped index into')
|
|
|
- parser.add_argument('--tcp-port', default=DEFAULT_TRANSPORT_TCP_PORT, type=int,
|
|
|
- help='The port to use as the minimum port for TCP communication')
|
|
|
- parser.add_argument('--http-port', default=DEFAULT_HTTP_TCP_PORT, type=int,
|
|
|
- help='The port to use as the minimum port for HTTP communication')
|
|
|
- cfg = parser.parse_args()
|
|
|
-
|
|
|
- if not os.path.exists(cfg.output_dir):
|
|
|
- parser.error('Output directory does not exist: %s' % cfg.output_dir)
|
|
|
-
|
|
|
- if not cfg.versions:
|
|
|
- # --all
|
|
|
- for bwc_index in glob.glob(os.path.join(cfg.output_dir, 'index-*.zip')):
|
|
|
- version = os.path.basename(bwc_index)[len('index-'):-len('.zip')]
|
|
|
- cfg.versions.append(version)
|
|
|
-
|
|
|
- return cfg
|
|
|
-
|
|
|
-def create_bwc_index(cfg, version):
|
|
|
- logging.info('--> Creating bwc index for %s' % version)
|
|
|
- release_dir = os.path.join(cfg.releases_dir, 'elasticsearch-%s' % version)
|
|
|
- if not os.path.exists(release_dir):
|
|
|
- raise RuntimeError('ES version %s does not exist in %s' % (version, cfg.releases_dir))
|
|
|
- snapshot_supported = not (version.startswith('0.') or version == '1.0.0.Beta1')
|
|
|
- tmp_dir = tempfile.mkdtemp()
|
|
|
-
|
|
|
- data_dir = os.path.join(tmp_dir, 'data')
|
|
|
- repo_dir = os.path.join(tmp_dir, 'repo')
|
|
|
- logging.info('Temp data dir: %s' % data_dir)
|
|
|
- logging.info('Temp repo dir: %s' % repo_dir)
|
|
|
-
|
|
|
- node = None
|
|
|
-
|
|
|
- try:
|
|
|
- node = start_node(version, release_dir, data_dir, repo_dir, cfg.tcp_port, cfg.http_port)
|
|
|
- client = create_client(cfg.http_port)
|
|
|
- index_name = 'index-%s' % version.lower()
|
|
|
- num_docs, capabilities = generate_index(client, version, index_name)
|
|
|
- if snapshot_supported:
|
|
|
- snapshot_index(client, version, repo_dir)
|
|
|
-
|
|
|
- # 10067: get a delete-by-query into the translog on upgrade. We must do
|
|
|
- # this after the snapshot, because it calls flush. Otherwise the index
|
|
|
- # will already have the deletions applied on upgrade.
|
|
|
- if version.startswith('0.') or version.startswith('1.'):
|
|
|
- delete_by_query(client, version, index_name, 'doc')
|
|
|
- reindex_docs(client, index_name, 'doc', min(100, num_docs), capabilities)
|
|
|
-
|
|
|
- shutdown_node(node)
|
|
|
- node = None
|
|
|
-
|
|
|
- compress_index(version, tmp_dir, cfg.output_dir)
|
|
|
- if snapshot_supported:
|
|
|
- compress_repo(version, tmp_dir, cfg.output_dir)
|
|
|
- finally:
|
|
|
-
|
|
|
- if node is not None:
|
|
|
- # This only happens if we've hit an exception:
|
|
|
- shutdown_node(node)
|
|
|
-
|
|
|
- shutil.rmtree(tmp_dir)
|
|
|
-
|
|
|
-def shutdown_node(node):
|
|
|
- logging.info('Shutting down node with pid %d', node.pid)
|
|
|
- node.kill() # don't use terminate otherwise we flush the translog
|
|
|
- node.wait()
|
|
|
-
|
|
|
-def parse_version(version):
|
|
|
- import re
|
|
|
- splitted = re.split('[.-]', version)
|
|
|
- if len(splitted) == 3:
|
|
|
- splitted = splitted + ['GA']
|
|
|
- splitted = [s.lower() for s in splitted]
|
|
|
- assert len(splitted) == 4;
|
|
|
- return splitted
|
|
|
-
|
|
|
-assert parse_version('5.0.0-alpha1') == parse_version('5.0.0-alpha1')
|
|
|
-assert parse_version('5.0.0-alpha1') < parse_version('5.0.0-alpha2')
|
|
|
-assert parse_version('5.0.0-alpha1') < parse_version('5.0.0-beta1')
|
|
|
-assert parse_version('5.0.0-beta1') < parse_version('5.0.0')
|
|
|
-assert parse_version('1.2.3') < parse_version('2.1.0')
|
|
|
-assert parse_version('1.2.3') < parse_version('1.2.4')
|
|
|
-assert parse_version('1.1.0') < parse_version('1.2.0')
|
|
|
-
|
|
|
-def main():
|
|
|
- logging.basicConfig(format='[%(levelname)s] [%(asctime)s] %(message)s', level=logging.INFO,
|
|
|
- datefmt='%Y-%m-%d %I:%M:%S %p')
|
|
|
- logging.getLogger('elasticsearch').setLevel(logging.ERROR)
|
|
|
- logging.getLogger('urllib3').setLevel(logging.WARN)
|
|
|
- cfg = parse_config()
|
|
|
- for version in cfg.versions:
|
|
|
- create_bwc_index(cfg, version)
|
|
|
-
|
|
|
-if __name__ == '__main__':
|
|
|
- try:
|
|
|
- main()
|
|
|
- except KeyboardInterrupt:
|
|
|
- print('Caught keyboard interrupt, exiting...')
|
Martijn van Groningen