Buildbot: Migrate package archive format for Linux from tar.bz2 to tar.xz
authorJens <JRottm>
Thu, 28 Nov 2019 10:40:59 +0000 (11:40 +0100)
committerSergey Sharybin <sergey.vfx@gmail.com>
Thu, 28 Nov 2019 10:41:25 +0000 (11:41 +0100)
xz compresses 25% better than bz2, reducing download times and server load.
The numbers:

blender-2.80-linux-glibc217-x86_64.tar.bz2 (release): 134 886 174 bytes
with xz:    96 181 604 bytes (-28.7%)
with xz -9: 93 871 548 bytes (-30.4%)

blender-2.81-7c1fbe24ca33-linux-glibc217-x86_64.tar.bz2 (beta): 173 600 363 bytes
with xz:    133 100 664 bytes (-23.3%)
with xz -9: 129 534 124 bytes (-25.4%)

xz also decompresses more than twice as fast as bz2, however compression needs
four times as long (on my 7-year-old laptop 3-4 minutes instead of <1).

Also xz has become more common than bz2, e.g. Debian/Ubuntu deb packages have
been xz-compressed for years, so the dpkg package manager as well as systemd
and grub all depend on liblzma being present, whereas bz2 is becoming more and
more optional.

Current Linux archives also include the UID/GID of whatever user account
happens to be used for building by the blender.org infrastructure. If someone
then installs these archives as root e.g. to /usr/local/... and doesn't pay
full attention the files remain owned by a regular user, which is a serious
security issue. This patch fixes that by setting the UID/GID to 0.

Differential Revision: https://developer.blender.org/D6138

build_files/buildbot/slave_pack.py
build_files/cmake/packaging.cmake
build_files/package_spec/build_archive.py

index 19dac2367627497f3939cb2e32fd0ef2d6a2121d..bbacedca0cef57ef400f9dbf38437ee058ac0c95 100644 (file)
@@ -64,7 +64,7 @@ def create_buildbot_upload_zip(builder, package_files):
         sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
         sys.exit(1)
 
-def create_tar_bz2(src, dest, package_name):
+def create_tar_xz(src, dest, package_name):
     # One extra to remove leading os.sep when cleaning root for package_root
     ln = len(src) + 1
     flist = list()
@@ -75,9 +75,20 @@ def create_tar_bz2(src, dest, package_name):
         flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
 
     import tarfile
-    package = tarfile.open(dest, 'w:bz2')
+
+    # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
+    # user compiled the package. If root then unpacks it to /usr/local/ you get
+    # a security issue.
+    def _fakeroot(tarinfo):
+        tarinfo.gid = 0
+        tarinfo.gname = "root"
+        tarinfo.uid = 0
+        tarinfo.uname = "root"
+        return tarinfo
+
+    package = tarfile.open(dest, 'w:xz', preset=9)
     for entry in flist:
-        package.add(entry[0], entry[1], recursive=False)
+        package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
     package.close()
 
 def cleanup_files(dirpath, extension):
@@ -163,11 +174,11 @@ def pack_linux(builder):
     # Construct package name
     platform_name = 'linux-' + blender_glibc + '-' + blender_arch
     package_name = get_package_name(builder, platform_name)
-    package_filename = package_name + ".tar.bz2"
+    package_filename = package_name + ".tar.xz"
 
-    print("Creating .tar.bz2 archive")
-    package_filepath = builder.install_dir + '.tar.bz2'
-    create_tar_bz2(builder.install_dir, package_filepath, package_name)
+    print("Creating .tar.xz archive")
+    package_filepath = builder.install_dir + '.tar.xz'
+    create_tar_xz(builder.install_dir, package_filepath, package_name)
 
     # Create buildbot_upload.zip
     create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
index 5ace42646c5a93779981e5a8e5314ca33a7235b8..0e530463659dae9a1fdba619f786f0a1b1e0d5ad 100644 (file)
@@ -135,7 +135,7 @@ elseif(UNIX)
 
   add_package_archive(
     "${PROJECT_NAME}-${BLENDER_VERSION}-${BUILD_REV}-${PACKAGE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}"
-    "tar.bz2")
+    "tar.xz")
 endif()
 
 unset(MAJOR_VERSION)
index 5ca2f319d8770a58e1889c80a353baa00300b786..58a970bca315a82278c6ce4401b3e2c2f8eb9e8d 100755 (executable)
@@ -51,8 +51,9 @@ try:
 
     if extension == 'zip':
         archive_cmd = ['zip', '-9', '-r', package_archive, package_dir]
-    elif extension == 'tar.bz2':
-        archive_cmd = ['tar', 'cjf', package_archive, package_dir]
+    elif extension == 'tar.xz':
+        archive_cmd = ['tar', '-cf', package_archive, '--owner=0', '--group=0',
+                       '--use-compress-program=xz -9', package_dir]
     else:
         sys.stderr.write('Unknown archive extension: ' + extension)
         sys.exit(-1)