Release scripts: Split prepare_release into two scripts

In order to reflect our RC release process, we need to separate
the prepare_release script into two separate scripts.

One script now updates the documentation. That one can be executed
anytime and needs to be pushed after that.

The other script updates the version in Version.java and all pom.xml
files, but does not commit anymore. This allows to create a non snapshot
version locally, run mvn deploy, push the artifacts into S3 and, upon
successful tests, simply release them on sonatype. This also allows for
updates, because the S3 snapshot will include the commitId in their repo
as already pushed before.

dev-tools/prepare_release.py → dev-tools/prepare_release_create_release_version.py

@@ -16,11 +16,8 @@
 
 # Prepare a release
 #
-# This script prepares a new release by creating two commits
-#
-# First commit: Update the Version.java to remove the snapshot bit
-# First commit: Remove the -SNAPSHOT suffix in all pom.xml files
-# Second commit: Update Documentation flags
+# 1. Update the Version.java to remove the snapshot bit
+# 2. Remove the -SNAPSHOT suffix in all pom.xml files
 #
 # USAGE:
 #
@@ -128,35 +125,6 @@ def find_release_version():
         return match.group(1)
     raise RuntimeError('Could not find release version in branch')
 
-# Stages the given files for the next git commit
-def add_pending_files(*files):
-  for file in files:
-    if file:
-      # print("Adding file: %s" % (file))
-      run('git add %s' % (file))
-
-# Executes a git commit with 'release [version]' as the commit message
-def commit_release(release):
-  run('git commit -m "Release: Change version from %s-SNAPSHOT to %s"' % (release, release))
-
-def commit_feature_flags(release):
-    run('git commit -m "Update Documentation Feature Flags [%s]"' % release)
-
-# Walks the given directory path (defaults to 'docs')
-# and replaces all 'coming[$version]' tags with
-# 'added[$version]'. This method only accesses asciidoc files.
-def update_reference_docs(release_version, path='docs'):
-  pattern = 'coming[%s' % (release_version)
-  replacement = 'added[%s' % (release_version)
-  pending_files = []
-  def callback(line):
-    return line.replace(pattern, replacement)
-  for root, _, file_names in os.walk(path):
-    for file_name in fnmatch.filter(file_names, '*.asciidoc'):
-      full_path = os.path.join(root, file_name)
-      if process_file(full_path, callback):
-        pending_files.append(os.path.join(root, file_name))
-  return pending_files
 
 if __name__ == "__main__":
   release_version = find_release_version()
@@ -169,18 +137,9 @@ if __name__ == "__main__":
   remove_maven_snapshot(pom_files, release_version)
   remove_version_snapshot(VERSION_FILE, release_version)
 
-  pending_files = pom_files
-  pending_files.append(VERSION_FILE)
-  add_pending_files(*pending_files) # expects var args use * to expand
-  commit_release(release_version)
-
-  pending_files = update_reference_docs(release_version)
-  # split commits for docs and version to enable easy cherry-picking
-  if pending_files:
-    add_pending_files(*pending_files) # expects var args use * to expand
-    commit_feature_flags(release_version)
-  else:
-    print('WARNING: no documentation references updates for release %s' % (release_version))
-
-  print('*** Done removing snapshot version. Run git push manually.')
+  print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.')
 
+  shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True)
+  print('')
+  print('*** To create a release candidate run: ')
+  print('  mvn clean deploy -Prelease -DskipTests -Dgpg.keyname="$GPG_KEY_ID" -Dgpg.passphrase="$GPG_PASSPHRASE" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Delasticsearch.s3.repository=s3://download.elasticsearch.org/elasticsearch/staging/%s' % (shortHash.decode('utf-8')))

dev-tools/prepare_release_update_documentation.py

@@ -0,0 +1,139 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance  with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on
+# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+# Prepare a release: Update the documentation and commit
+#
+# USAGE:
+#
+# python3 ./dev-tools/prepare_release_update_documentation.py
+#
+# Note: Ensure the script is run from the root directory
+#       This script needs to be run and then pushed,
+#       before proceeding with prepare_release_create-release-version.py
+#       on your build VM
+#
+
+import fnmatch
+import subprocess
+import tempfile
+import re
+import os
+import shutil
+
+def run(command):
+  if os.system('%s' % (command)):
+    raise RuntimeError('    FAILED: %s' % (command))
+
+def ensure_checkout_is_clean():
+  # Make sure no local mods:
+  s = subprocess.check_output('git diff --shortstat', shell=True)
+  if len(s) > 0:
+    raise RuntimeError('git diff --shortstat is non-empty: got:\n%s' % s)
+
+  # Make sure no untracked files:
+  s = subprocess.check_output('git status', shell=True).decode('utf-8', errors='replace')
+  if 'Untracked files:' in s:
+    raise RuntimeError('git status shows untracked files: got:\n%s' % s)
+
+  # Make sure we have all changes from origin:
+  if 'is behind' in s:
+    raise RuntimeError('git status shows not all changes pulled from origin; try running "git pull origin" in this branch: got:\n%s' % (s))
+
+  # Make sure we no local unpushed changes (this is supposed to be a clean area):
+  if 'is ahead' in s:
+    raise RuntimeError('git status shows local commits; try running "git fetch origin", "git checkout ", "git reset --hard origin/" in this branch: got:\n%s' % (s))
+
+# Reads the given file and applies the
+# callback to it. If the callback changed
+# a line the given file is replaced with
+# the modified input.
+def process_file(file_path, line_callback):
+  fh, abs_path = tempfile.mkstemp()
+  modified = False
+  with open(abs_path,'w', encoding='utf-8') as new_file:
+    with open(file_path, encoding='utf-8') as old_file:
+      for line in old_file:
+        new_line = line_callback(line)
+        modified = modified or (new_line != line)
+        new_file.write(new_line)
+  os.close(fh)
+  if modified:
+    #Remove original file
+    os.remove(file_path)
+    #Move new file
+    shutil.move(abs_path, file_path)
+    return True
+  else:
+    # nothing to do - just remove the tmp file
+    os.remove(abs_path)
+    return False
+
+# Checks the pom.xml for the release version.
+# This method fails if the pom file has no SNAPSHOT version set ie.
+# if the version is already on a release version we fail.
+# Returns the next version string ie. 0.90.7
+def find_release_version():
+  with open('pom.xml', encoding='utf-8') as file:
+    for line in file:
+      match = re.search(r'<version>(.+)-SNAPSHOT</version>', line)
+      if match:
+        return match.group(1)
+    raise RuntimeError('Could not find release version in branch')
+
+# Stages the given files for the next git commit
+def add_pending_files(*files):
+  for file in files:
+    if file:
+      # print("Adding file: %s" % (file))
+      run('git add %s' % (file))
+
+# Updates documentation feature flags
+def commit_feature_flags(release):
+    run('git commit -m "Update Documentation Feature Flags [%s]"' % release)
+
+# Walks the given directory path (defaults to 'docs')
+# and replaces all 'coming[$version]' tags with
+# 'added[$version]'. This method only accesses asciidoc files.
+def update_reference_docs(release_version, path='docs'):
+  pattern = 'coming[%s' % (release_version)
+  replacement = 'added[%s' % (release_version)
+  pending_files = []
+  def callback(line):
+    return line.replace(pattern, replacement)
+  for root, _, file_names in os.walk(path):
+    for file_name in fnmatch.filter(file_names, '*.asciidoc'):
+      full_path = os.path.join(root, file_name)
+      if process_file(full_path, callback):
+        pending_files.append(os.path.join(root, file_name))
+  return pending_files
+
+if __name__ == "__main__":
+  release_version = find_release_version()
+
+  print('*** Preparing release version documentation: [%s]' % release_version)
+
+  ensure_checkout_is_clean()
+
+  pending_files = update_reference_docs(release_version)
+  
+  if pending_files:
+    add_pending_files(*pending_files) # expects var args use * to expand
+    commit_feature_flags(release_version)
+  else:
+    print('WARNING: no documentation references updates for release %s' % (release_version))
+
+  print('*** Done.')
+