Browse Source

integrate standalone schroot setup/update scripts into the build script

A mini-DSL is used to specify steps for setting up of a target distribution,
which automatically sets it up for both the i386 and amd64 architectures.
This also allows support for other distributions to be added very easily.
Ashish Kulkarni 11 years ago
parent
commit
1fbe69f04b
4 changed files with 123 additions and 157 deletions
  1. 7 6
      INSTALL.md
  2. 116 0
      scripts/build.py
  3. 0 114
      scripts/setup-linux.sh
  4. 0 37
      scripts/update-linux.sh

+ 7 - 6
INSTALL.md

@@ -7,13 +7,14 @@ If you are on Windows, make sure that you are cloning in a location without spac
 Prerequisites: Linux
 --------------------
 
-Building is currently supported only on recent 64-bit Debian/Ubuntu releases. Please run ```scripts/setup-linux.sh``` as root, which will create 4 chroot environments and configure them for building properly (approx 2GB will need to be downloaded):
- * Debian-wheezy i386  (should work on Ubuntu 12.04 LTS and later)
- * Debian-wheezy amd64 (should work on Ubuntu 12.04 LTS and later)
- * CentOS-5 i386  (should work on systems where older GLIBC is present)
- * CentOS-5 amd64 (should work on systems where older GLIBC is present)
+Building is currently supported only on recent 64-bit Debian/Ubuntu releases. All binaries are produced in a self-contained chroot environment for the target distribution, so you will need to setup it up first by running ```scripts/build.py```. The following distributions are currently supported:
 
-It will also setup a MinGW-w64 toolchain, which can cross-compile Windows binaries from Linux -- useful for running on Windows XP/Windows 2003, which are not supported by default when compiling with MSVC 2013. 
+Distribution   | Command for Setup
+------------   | -----------------
+Debian Wheezy  | ```scripts/build.py setup-schroot-wheezy```
+CentOS 5       | ```scripts/build.py setup-schroot-centos5```
+
+Please note a MinGW-w64 toolchain (which can cross-compile Windows binaries from Linux) is setup by default -- it is useful for targetting Windows XP/Windows 2003, which are not supported by default when compiling with MSVC 2013.
 
 Prerequisites: Windows
 ----------------------

+ 116 - 0
scripts/build.py

@@ -167,6 +167,9 @@ BUILDERS = {
     'msvc2013-win64':        'msvc',
     'msvc-winsdk71-win32':   'msvc_winsdk71',
     'msvc-winsdk71-win64':   'msvc_winsdk71',
+    'setup-schroot-centos5': 'setup_schroot',
+    'setup-schroot-wheezy':  'setup_schroot',
+    'update-all-schroots':   'update_schroot',
     'centos5-i386':          'linux_schroot',
     'centos5-amd64':         'linux_schroot',
     'wheezy-i386':           'linux_schroot',
@@ -175,6 +178,35 @@ BUILDERS = {
     'mingw-w64-cross-win64': 'mingw64_cross'
 }
 
+HOST_PACKAGES = ['git-core', 'xz-utils', 'build-essential', 'mingw-w64', 'nsis', 'debootstrap', 'schroot', 'rinse']
+CHROOT_SETUP  = {
+    'wheezy': [
+        ('debootstrap', 'wheezy', 'http://ftp.us.debian.org/debian/'),
+        ('write_file', 'etc/apt/sources.list', """
+deb     http://ftp.debian.org/debian/ wheezy         main contrib non-free
+deb     http://ftp.debian.org/debian/ wheezy-updates main contrib non-free
+deb     http://security.debian.org/   wheezy/updates main contrib non-free
+deb-src http://ftp.debian.org/debian/ wheezy         main contrib non-free
+deb-src http://ftp.debian.org/debian/ wheezy-updates main contrib non-free
+deb-src http://security.debian.org/   wheezy/updates main contrib non-free"""),
+        ('shell', 'apt-get update'),
+        ('shell', 'apt-get dist-upgrade --assume-yes'),
+        ('shell', 'apt-get install --assume-yes xz-utils'),
+        ('shell', 'apt-get build-dep --assume-yes libqt4-core'),
+        ('write_file', 'update.sh', 'apt-get update\napt-get dist-upgrade --assume-yes\n'),
+        ('schroot_conf', 'Debian Wheezy')
+    ],
+
+    'centos5': [
+        ('rinse', 'centos-5'),
+        ('shell', 'yum update -y'),
+        ('append_file:amd64', 'etc/yum.conf', 'exclude = *.i?86\n'),
+        ('shell', 'yum install -y gcc gcc-c++ make qt4-devel openssl-devel diffutils perl xz'),
+        ('write_file', 'update.sh', 'yum update -y\n'),
+        ('schroot_conf', 'CentOS 5')
+    ]
+}
+
 # --------------------------------------------------------------- HELPERS
 
 import os, sys, subprocess, shutil, fnmatch, multiprocessing
@@ -257,6 +289,90 @@ def build_openssl(config, basedir):
 
     return OPENSSL['build'][cfg]['os_libs']
 
+# --------------------------------------------------------------- Linux chroot
+
+def check_setup_schroot(config):
+    import platform
+    if not sys.platform.startswith('linux') or \
+       not platform.architecture()[0].startswith('64') or \
+       platform.linux_distribution()[0] not in ['Ubuntu', 'Debian']:
+        error('This can only be run on a 64-bit Debian/Ubuntu distribution, aborting.')
+
+    if os.geteuid() != 0:
+        error('This script must be run as root.')
+
+    try:
+        login = subprocess.check_output(['logname'], stderr=subprocess.STDOUT).strip()
+        if login == 'root':
+            error('Please run via sudo to determine login for which schroot access is to be given.')
+    except subprocess.CalledProcessError:
+        error('Unable to determine the login for which schroot access is to be given.')
+
+def build_setup_schroot(config, basedir):
+    shell('apt-get update')
+    shell('apt-get install --assume-yes %s' % (' '.join(HOST_PACKAGES)))
+
+    login  = subprocess.check_output(['logname'], stderr=subprocess.STDOUT).strip()
+    chroot = config[1+config.rindex('-'):]
+    for arch in ['amd64', 'i386']:
+        print '******************* %s-%s' % (chroot, arch)
+        root_dir = '/opt/wkhtmltopdf-build/%s-%s' % (chroot, arch)
+        rmdir(root_dir)
+        mkdir_p(root_dir)
+        for command in CHROOT_SETUP[chroot]:
+            # handle architecture-specific commands
+            name = command[0]
+            if ':' in name:
+                if name[1+name.rindex(':'):] != arch:
+                    continue
+                else:
+                    name = name[:name.rindex(':')]
+
+            # handle commands
+            if name == 'debootstrap':
+                shell('debootstrap --arch=%(arch)s --variant=buildd %(distro)s %(dir)s %(url)s' % {
+                    'arch': arch, 'dir': root_dir, 'distro': command[1], 'url': command[2] })
+            elif name == 'rinse':
+                cmd = (arch == 'i386' and 'linux32 rinse' or 'rinse')
+                shell('%s --arch %s --distribution %s --directory %s' % (cmd, arch, command[1], root_dir))
+            elif name == 'shell':
+                cmd = (arch == 'i386' and 'linux32 chroot' or 'chroot')
+                shell('%s %s %s' % (cmd, root_dir, command[1]))
+            elif name == 'write_file':
+                open(os.path.join(root_dir, command[1]), 'w').write(command[2].strip())
+            elif name == 'append_file':
+                open(os.path.join(root_dir, command[1]), 'a').write(command[2].strip())
+            elif name == 'schroot_conf':
+                cfg = open('/etc/schroot/chroot.d/wkhtmltopdf-%s-%s' % (chroot, arch), 'w')
+                cfg.write('[wkhtmltopdf-%s-%s]\n' % (chroot, arch))
+                cfg.write('type=directory\ndirectory=%s/\n' % root_dir)
+                cfg.write('description=%s %s for wkhtmltopdf\n' % (command[1], arch))
+                cfg.write('users=%s\nroot-users=root\n' % login)
+                if arch == 'i386':
+                    cfg.write('personality=linux32\n')
+                cfg.close()
+
+def check_update_schroot(config):
+    import platform
+    if not sys.platform.startswith('linux') or \
+       not platform.architecture()[0].startswith('64') or \
+       platform.linux_distribution()[0] not in ['Ubuntu', 'Debian']:
+        error('This can only be run on a 64-bit Debian/Ubuntu distribution, aborting.')
+
+    if os.geteuid() != 0:
+        error('This script must be run as root.')
+
+    try:
+        subprocess.check_output(['schroot', '--list'], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        error('Unable to determine the list of available schroots.')
+
+def build_update_schroot(config, basedir):
+    names = subprocess.check_output(['schroot', '--list'], stderr=subprocess.STDOUT).strip()
+    for name in names.split('\n'):
+        print '******************* %s' % name[name.index('wkhtmltopdf-'):]
+        shell('schroot -c %s -- /bin/bash /update.sh' % name[name.index('wkhtmltopdf-'):])
+
 # --------------------------------------------------------------- MSVC (2008-2013)
 
 MSVC_LOCATION = {

+ 0 - 114
scripts/setup-linux.sh

@@ -1,114 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014 wkhtmltopdf authors
-#
-# This file is part of wkhtmltopdf.
-#
-# wkhtmltopdf is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# wkhtmltopdf is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with wkhtmltopdf.  If not, see <http:#www.gnu.org/licenses/>.
-
-if [ "$(id -u)" != "0" ]; then
-   echo "This script must be run as root" 1>&2
-   exit 1
-fi
-
-if [ $# -ne 1 ]; then
-    echo "Usage: scripts/setup-linux.sh [user-to-grant-schroot-access]"
-    exit 1
-fi
-
-# install required packages
-apt-get update
-apt-get install --assume-yes git-core xz-utils build-essential mingw-w64 nsis debootstrap schroot rinse
-
-# create the directory which will hold the chroot environments
-if [ -d /srv/chroot-wkhtmltopdf ]; then
-    rm -fr /srv/chroot-wkhtmltopdf
-fi
-mkdir -p /srv/chroot-wkhtmltopdf
-
-# create chroots for Debian Wheezy
-debootstrap --arch=i386  --variant=buildd wheezy /srv/chroot-wkhtmltopdf/wheezy-i386  http://ftp.us.debian.org/debian/
-debootstrap --arch=amd64 --variant=buildd wheezy /srv/chroot-wkhtmltopdf/wheezy-amd64 http://ftp.us.debian.org/debian/
-
-# update packages and install build dependencies
-cat > /srv/chroot-wkhtmltopdf/wheezy-amd64/etc/apt/sources.list <<EOF
-deb     http://ftp.debian.org/debian/ wheezy         main contrib non-free
-deb     http://ftp.debian.org/debian/ wheezy-updates main contrib non-free
-deb     http://security.debian.org/   wheezy/updates main contrib non-free
-deb-src http://ftp.debian.org/debian/ wheezy         main contrib non-free
-deb-src http://ftp.debian.org/debian/ wheezy-updates main contrib non-free
-deb-src http://security.debian.org/   wheezy/updates main contrib non-free
-EOF
-cp /srv/chroot-wkhtmltopdf/wheezy-amd64/etc/apt/sources.list /srv/chroot-wkhtmltopdf/wheezy-i386/etc/apt/sources.list
-chroot         /srv/chroot-wkhtmltopdf/wheezy-amd64/ apt-get update
-linux32 chroot /srv/chroot-wkhtmltopdf/wheezy-i386/  apt-get update
-chroot         /srv/chroot-wkhtmltopdf/wheezy-amd64/ apt-get dist-upgrade --assume-yes
-linux32 chroot /srv/chroot-wkhtmltopdf/wheezy-i386/  apt-get dist-upgrade --assume-yes
-chroot         /srv/chroot-wkhtmltopdf/wheezy-amd64/ apt-get install --assume-yes xz-utils
-linux32 chroot /srv/chroot-wkhtmltopdf/wheezy-i386/  apt-get install --assume-yes xz-utils
-chroot         /srv/chroot-wkhtmltopdf/wheezy-amd64/ apt-get build-dep --assume-yes libqt4-core
-linux32 chroot /srv/chroot-wkhtmltopdf/wheezy-i386/  apt-get build-dep --assume-yes libqt4-core
-
-# create chroots for Centos 5
-linux32 rinse --arch i386  --distribution centos-5 --directory /srv/chroot-wkhtmltopdf/centos-5-i386
-rinse         --arch amd64 --distribution centos-5 --directory /srv/chroot-wkhtmltopdf/centos-5-amd64
-
-# update packages and install development tools and build dependencies
-wget http://dl.fedoraproject.org/pub/epel/5/i386/epel-release-5-4.noarch.rpm
-cp epel-release-5-4.noarch.rpm /srv/chroot-wkhtmltopdf/centos-5-amd64
-mv epel-release-5-4.noarch.rpm /srv/chroot-wkhtmltopdf/centos-5-i386
-chroot         /srv/chroot-wkhtmltopdf/centos-5-amd64 yum update -y
-linux32 chroot /srv/chroot-wkhtmltopdf/centos-5-i386  yum update -y
-chroot         /srv/chroot-wkhtmltopdf/centos-5-amd64 rpm -i /epel-release-5-4.noarch.rpm
-linux32 chroot /srv/chroot-wkhtmltopdf/centos-5-i386  rpm -i /epel-release-5-4.noarch.rpm
-echo "exclude = *.i?86">>/srv/chroot-wkhtmltopdf/centos-5-amd64/etc/yum.conf
-chroot         /srv/chroot-wkhtmltopdf/centos-5-amd64 yum install -y gcc gcc-c++ make qt4-devel openssl-devel diffutils perl xz
-linux32 chroot /srv/chroot-wkhtmltopdf/centos-5-i386  yum install -y gcc gcc-c++ make qt4-devel openssl-devel diffutils perl xz
-rm /srv/chroot-wkhtmltopdf/centos-5-i386/epel-release-5-4.noarch.rpm
-rm /srv/chroot-wkhtmltopdf/centos-5-amd64/epel-release-5-4.noarch.rpm
-
-# create schroot configuration
-cat > /etc/schroot/chroot.d/wkhtmltopdf <<EOF
-[wkhtmltopdf-wheezy-i386]
-type=directory
-directory=/srv/chroot-wkhtmltopdf/wheezy-i386/
-description=Debian Wheezy i386 for wkhtmltopdf
-users=$1
-root-users=root
-personality=linux32
-
-[wkhtmltopdf-wheezy-amd64]
-type=directory
-directory=/srv/chroot-wkhtmltopdf/wheezy-amd64/
-description=Debian Wheezy amd64 for wkhtmltopdf
-users=$1
-root-users=root
-
-[wkhtmltopdf-centos5-i386]
-type=directory
-directory=/srv/chroot-wkhtmltopdf/centos-5-i386/
-description=CentOS 5 i386 for wkhtmltopdf
-users=$1
-root-users=root
-personality=linux32
-
-[wkhtmltopdf-centos5-amd64]
-type=directory
-directory=/srv/chroot-wkhtmltopdf/centos-5-amd64/
-description=CentOS 5 amd64 for wkhtmltopdf
-users=$1
-root-users=root
-EOF
-
-echo "Environments set up successfully."

+ 0 - 37
scripts/update-linux.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2014 wkhtmltopdf authors
-#
-# This file is part of wkhtmltopdf.
-#
-# wkhtmltopdf is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# wkhtmltopdf is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with wkhtmltopdf.  If not, see <http:#www.gnu.org/licenses/>.
-
-if [ "$(id -u)" != "0" ]; then
-   echo "This script must be run as root" 1>&2
-   exit 1
-fi
-
-# update the host machine
-apt-get update
-apt-get dist-upgrade --assume-yes
-
-# update the wheezy chroots
-schroot -c wkhtmltopdf-wheezy-amd64 -- apt-get update
-schroot -c wkhtmltopdf-wheezy-amd64 -- apt-get dist-upgrade --assume-yes
-schroot -c wkhtmltopdf-wheezy-i386  -- apt-get update
-schroot -c wkhtmltopdf-wheezy-i386  -- apt-get dist-upgrade --assume-yes
-
-# update the centos 5 chroots
-schroot -c wkhtmltopdf-centos5-amd64 -- yum update -y
-schroot -c wkhtmltopdf-centos5-i386  -- yum update -y