marko10_000
/
flatpak-builder
forked from Mirrors/flatpak-builder
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
flatpak-builder/src/builder-post-process.c

562 lines
18 KiB
C
Raw Permalink Blame History

/* builder-post-process.c
*
* Copyright (C) 2017 Red Hat, Inc
*
* This file is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This file is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Authors:
* Alexander Larsson <alexl@redhat.com>
*/
#include "config.h"
#include <string.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/statfs.h>
#include <gio/gio.h>
#include "libglnx/libglnx.h"
#include "builder-flatpak-utils.h"
#include "builder-utils.h"
#include "builder-post-process.h"
static gboolean
invalidate_old_python_compiled (const char *path,
const char *rel_path,
GError **error)
{
struct stat stbuf;
g_autofree char *pyc = NULL;
g_autofree char *pyo = NULL;
g_autofree char *dir = NULL;
g_autofree char *py3dir = NULL;
g_autofree char *pyfilename = NULL;
g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
/* This is a python file, not a .py[oc]. If it changed (mtime != 0) then
* this needs to invalidate any old (mtime == 0) .py[oc] files that could refer to it.
*/
if (lstat (path, &stbuf) != 0)
{
g_warning ("Can't stat %s", rel_path);
return TRUE;
}
if (stbuf.st_mtime == OSTREE_TIMESTAMP)
return TRUE; /* Previously handled .py */
pyc = g_strconcat (path, "c", NULL);
if (lstat (pyc, &stbuf) == 0 &&
stbuf.st_mtime == OSTREE_TIMESTAMP)
{
g_print ("Removing stale file %sc", rel_path);
if (unlink (pyc) != 0)
g_warning ("Unable to delete %s", pyc);
}
pyo = g_strconcat (path, "o", NULL);
if (lstat (pyo, &stbuf) == 0 &&
stbuf.st_mtime == OSTREE_TIMESTAMP)
{
g_print ("Removing stale file %so", rel_path);
if (unlink (pyo) != 0)
g_warning ("Unable to delete %s", pyo);
}
/* Handle python3 which is in a __pycache__ subdir */
pyfilename = g_path_get_basename (path);
pyfilename[strlen (pyfilename) - 2] = 0; /* skip "py" */
dir = g_path_get_dirname (path);
py3dir = g_build_filename (dir, "__pycache__", NULL);
if (glnx_dirfd_iterator_init_at (AT_FDCWD, py3dir, FALSE, &dfd_iter, NULL))
{
struct dirent *dent;
while (glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, NULL, NULL) &&
dent != NULL)
{
if (!(g_str_has_suffix (dent->d_name, ".pyc") ||
g_str_has_suffix (dent->d_name, ".pyo")))
continue;
if (!g_str_has_prefix (dent->d_name, pyfilename))
continue;
if (fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW) == 0 &&
stbuf.st_mtime == OSTREE_TIMESTAMP)
{
g_print ("Removing stale file %s/__pycache__/%s", rel_path, dent->d_name);
if (unlinkat (dfd_iter.fd, dent->d_name, 0))
g_warning ("Unable to delete %s", dent->d_name);
}
}
}
return TRUE;
}
/* We need to read at least 12 bytes to get both magic + optional flags + mtime */
#define PYTHON_HEADER_SIZE 12
static gboolean
fixup_python_time_stamp (const char *path,
const char *rel_path,
GError **error)
{
glnx_fd_close int fd = -1;
g_auto(GLnxTmpfile) tmpf = { 0 };
guint8 buffer[PYTHON_HEADER_SIZE];
ssize_t res;
guint32 pyc_mtime;
guint32 magic, header_flag;
gsize mtime_offset;
g_autofree char *py_path = NULL;
struct stat stbuf;
gboolean remove_pyc = FALSE;
g_autofree char *path_basename = g_path_get_basename (path);
g_autofree char *dir = g_path_get_dirname (path);
g_autofree char *dir_basename = g_path_get_basename (dir);
fd = open (path, O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
if (fd == -1)
{
g_warning ("Can't open %s", rel_path);
return TRUE;
}
res = pread (fd, buffer, PYTHON_HEADER_SIZE, 0);
if (res != PYTHON_HEADER_SIZE)
{
g_warning ("Short read for %s", rel_path);
return TRUE;
}
if (buffer[2] != 0x0d || buffer[3] != 0x0a)
{
g_debug ("Not matching python magic: %s", rel_path);
return TRUE;
}
magic = buffer[0] + (buffer[1] << 8);
/* All magic listed here: https://github.com/python/cpython/blob/HEAD/Lib/importlib/_bootstrap_external.py#L167
* 3392 is the first (3.7) which added an extra flags field in the header.
* 20121 is the first higher major listed (1.5), and all other non-py3 ones are higher.
*/
if (magic >= 3392 && magic < 20121)
{
/* From the spec:
* The pyc header currently consists of 3 32-bit words. We will expand it to 4.
* The first word will continue to be the magic number, versioning the bytecode and pyc format.
* The second word, conceptually the new word, will be a bit field.
* The interpretation of the rest of the header and invalidation behavior of the pyc depends on the contents of the bit field.
*/
header_flag =
(buffer[4] << 8*0) |
(buffer[5] << 8*1) |
(buffer[6] << 8*2) |
(buffer[7] << 8*3);
/* If the bit field is 0, the pyc is a traditional timestamp-based pyc. I.e., the third
and forth words will be the timestamp and file size respectively, and invalidation
will be done by comparing the metadata of the source file with that in the header. */
if (header_flag != 0)
{
/* Non-mtime based verification, like hash if low bit is 1,
* or other future added methods. No need to do anything*/
return TRUE;
}
mtime_offset = 8;
}
else
{
mtime_offset = 4;
}
pyc_mtime =
(buffer[mtime_offset+0] << 8*0) |
(buffer[mtime_offset+1] << 8*1) |
(buffer[mtime_offset+2] << 8*2) |
(buffer[mtime_offset+3] << 8*3);
if (strcmp (dir_basename, "__pycache__") == 0)
{
/* Python3 */
g_autofree char *base = g_strdup (path_basename);
g_autofree char *real_dir = g_path_get_dirname (dir);
g_autofree char *py_basename = NULL;
char *dot;
dot = strrchr (base, '.');
if (dot == NULL)
return TRUE;
*dot = 0;
dot = strrchr (base, '.');
if (dot == NULL)
return TRUE;
*dot = 0;
py_basename = g_strconcat (base, ".py", NULL);
py_path = g_build_filename (real_dir, py_basename, NULL);
}
else
{
/* Python2 */
py_path = g_strndup (path, strlen (path) - 1);
}
/* Here we found a .pyc (or .pyo) file and a possible .py file that apply for it.
* There are several possible cases wrt their mtimes:
*
* py not existing: pyc is stale, remove it
* pyc mtime == 0: (.pyc is from an old commited module)
* py mtime == 0: Do nothing, already correct
* py mtime != 0: The py changed in this module, remove pyc
* pyc mtime != 0: (.pyc changed this module, or was never rewritten in base layer)
* py mtime == 0: Shouldn't happen in flatpak-builder, but could be an un-rewritten ctime lower layer, assume it matches and update timestamp
* py mtime != pyc mtime: new pyc doesn't match last py written in this module, remove it
* py mtime == pyc mtime: These match, but the py will be set to mtime 0 by ostree, so update timestamp in pyc.
*/
if (lstat (py_path, &stbuf) != 0)
{
/* pyc file without .py file, this happens for binary-only deployments.
* Accept it as-is. */
return TRUE;
}
else if (pyc_mtime == OSTREE_TIMESTAMP)
{
if (stbuf.st_mtime == OSTREE_TIMESTAMP)
return TRUE; /* Previously handled pyc */
remove_pyc = TRUE;
}
else /* pyc_mtime != 0 */
{
if (pyc_mtime != stbuf.st_mtime && stbuf.st_mtime != OSTREE_TIMESTAMP)
remove_pyc = TRUE;
/* else change mtime */
}
if (remove_pyc)
{
g_print ("Removing stale python bytecode file %s\n", rel_path);
if (unlink (path) != 0)
g_warning ("Unable to delete %s", rel_path);
return TRUE;
}
if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dir,
O_RDWR | O_CLOEXEC | O_NOFOLLOW,
&tmpf,
error))
return FALSE;
if (glnx_regfile_copy_bytes (fd, tmpf.fd, (off_t)-1) < 0)
return glnx_throw_errno_prefix (error, "copyfile");
/* Change to mtime 0 which is what ostree uses for checkouts */
buffer[mtime_offset+0] = OSTREE_TIMESTAMP;
buffer[mtime_offset+1] = buffer[mtime_offset+2] = buffer[mtime_offset+3] = 0;
res = pwrite (tmpf.fd, buffer, PYTHON_HEADER_SIZE, 0);
if (res != PYTHON_HEADER_SIZE)
{
glnx_set_error_from_errno (error);
return FALSE;
}
if (!glnx_link_tmpfile_at (&tmpf,
GLNX_LINK_TMPFILE_REPLACE,
AT_FDCWD,
path,
error))
return FALSE;
g_print ("Fixed up header mtime for %s\n", rel_path);
/* The mtime will be zeroed on cache commit. We don't want to do that now, because multiple
files could reference one .py file and we need the mtimes to match for them all */
return TRUE;
}
static gboolean
builder_post_process_python_time_stamp (GFile *app_dir,
GPtrArray *changed,
GError **error)
{
int i;
for (i = 0; i < changed->len; i++)
{
const char *rel_path = (char *) g_ptr_array_index (changed, i);
g_autoptr(GFile) file = NULL;
g_autofree char *path = NULL;
struct stat stbuf;
if (!(g_str_has_suffix (rel_path, ".py") ||
g_str_has_suffix (rel_path, ".pyc") ||
g_str_has_suffix (rel_path, ".pyo")))
continue;
file = g_file_resolve_relative_path (app_dir, rel_path);
path = g_file_get_path (file);
if (lstat (path, &stbuf) == -1)
continue;
if (!S_ISREG (stbuf.st_mode))
continue;
if (g_str_has_suffix (rel_path, ".py"))
{
if (!invalidate_old_python_compiled (path, rel_path, error))
return FALSE;
}
else
{
if (!fixup_python_time_stamp (path, rel_path, error))
return FALSE;
}
}
return TRUE;
}
static gboolean
builder_post_process_strip (GFile *app_dir,
GPtrArray *changed,
GError **error)
{
int i;
for (i = 0; i < changed->len; i++)
{
const char *rel_path = (char *) g_ptr_array_index (changed, i);
g_autoptr(GFile) file = g_file_resolve_relative_path (app_dir, rel_path);
g_autofree char *path = g_file_get_path (file);
gboolean is_shared, is_stripped;
if (!is_elf_file (path, &is_shared, &is_stripped))
continue;
if (is_stripped)
continue;
g_print ("stripping: %s\n", rel_path);
if (is_shared)
{
if (!strip (error, "--remove-section=.comment", "--remove-section=.note", "--strip-unneeded", path, NULL))
return FALSE;
}
else
{
if (!strip (error, "--remove-section=.comment", "--remove-section=.note", path, NULL))
return FALSE;
}
}
return TRUE;
}
static gboolean
builder_post_process_debuginfo (GFile *app_dir,
GPtrArray *changed,
BuilderPostProcessFlags flags,
BuilderContext *context,
GError **error)
{
g_autofree char *app_dir_path = g_file_get_path (app_dir);
int j;
for (j = 0; j < changed->len; j++)
{
const char *rel_path = (char *) g_ptr_array_index (changed, j);
g_autoptr(GFile) file = g_file_resolve_relative_path (app_dir, rel_path);
g_autofree char *path = g_file_get_path (file);
g_autofree char *debug_path = NULL;
g_autofree char *real_debug_path = NULL;
g_autofree char *rel_path_dir = g_path_get_dirname (rel_path);
g_autofree char *filename = g_path_get_basename (rel_path);
g_autofree char *filename_debug = g_strconcat (filename, ".debug", NULL);
g_autofree char *debug_dir = NULL;
g_autofree char *source_dir_path = NULL;
g_autoptr(GFile) source_dir = NULL;
g_autofree char *real_debug_dir = NULL;
gboolean is_shared, is_stripped;
if (!is_elf_file (path, &is_shared, &is_stripped))
continue;
if (is_stripped)
continue;
if (g_str_has_prefix (rel_path_dir, "files/"))
{
debug_dir = g_build_filename (app_dir_path, "files/lib/debug", rel_path_dir + strlen ("files/"), NULL);
real_debug_dir = g_build_filename ("/app/lib/debug", rel_path_dir + strlen ("files/"), NULL);
source_dir_path = g_build_filename (app_dir_path, "files/lib/debug/source", NULL);
}
else if (g_str_has_prefix (rel_path_dir, "usr/"))
{
debug_dir = g_build_filename (app_dir_path, "usr/lib/debug", rel_path_dir, NULL);
real_debug_dir = g_build_filename ("/usr/lib/debug", rel_path_dir, NULL);
source_dir_path = g_build_filename (app_dir_path, "usr/lib/debug/source", NULL);
}
if (debug_dir)
{
const char *builddir;
g_autoptr(GError) local_error = NULL;
g_auto(GStrv) file_refs = NULL;
if (g_mkdir_with_parents (debug_dir, 0755) != 0)
{
glnx_set_error_from_errno (error);
return FALSE;
}
source_dir = g_file_new_for_path (source_dir_path);
if (g_mkdir_with_parents (source_dir_path, 0755) != 0)
{
glnx_set_error_from_errno (error);
return FALSE;
}
if (builder_context_get_build_runtime (context))
builddir = "/run/build-runtime/";
else
builddir = "/run/build/";
debug_path = g_build_filename (debug_dir, filename_debug, NULL);
real_debug_path = g_build_filename (real_debug_dir, filename_debug, NULL);
file_refs = builder_get_debuginfo_file_references (path, &local_error);
if (file_refs == NULL)
{
g_warning ("%s", local_error->message);
}
else
{
GFile *build_dir = builder_context_get_build_dir (context);
int i;
for (i = 0; file_refs[i] != NULL; i++)
{
if (g_str_has_prefix (file_refs[i], builddir))
{
const char *relative_path = file_refs[i] + strlen (builddir);
g_autoptr(GFile) src = g_file_resolve_relative_path (build_dir, relative_path);
g_autoptr(GFile) dst = g_file_resolve_relative_path (source_dir, relative_path);
g_autoptr(GFile) dst_parent = g_file_get_parent (dst);
GFileType file_type;
if (!flatpak_mkdir_p (dst_parent, NULL, error))
return FALSE;
file_type = g_file_query_file_type (src, 0, NULL);
if (file_type == G_FILE_TYPE_DIRECTORY)
{
if (!flatpak_mkdir_p (dst, NULL, error))
return FALSE;
}
else if (file_type == G_FILE_TYPE_REGULAR)
{
/* Make sure the target is gone, because g_file_copy does
truncation on hardlinked destinations */
(void)g_file_delete (dst, NULL, NULL);
if (!g_file_copy (src, dst,
G_FILE_COPY_OVERWRITE,
NULL, NULL, NULL, error))
return FALSE;
}
}
}
}
/* Some files are hardlinked and eu-strip modifies in-place,
which breaks rofiles-fuse. Unlink them */
if (!flatpak_break_hardlink (file, error))
return FALSE;
if (flags & BUILDER_POST_PROCESS_FLAGS_DEBUGINFO_COMPRESSION)
{
g_autoptr(GError) my_error = NULL;
g_print ("compressing debuginfo in: %s\n", path);
if (!eu_elfcompress (&my_error, "-t", "zlib-gnu", "-v", path, NULL))
{
if (g_error_matches (my_error, G_SPAWN_ERROR, G_SPAWN_ERROR_NOENT))
g_print ("Warning: eu-elfcompress not installed, will not compress debuginfo\n");
else
{
g_propagate_error (error, g_steal_pointer (&my_error));
return FALSE;
}
}
}
g_print ("stripping %s to %s\n", path, debug_path);
if (!eu_strip (error, "--remove-comment", "--reloc-debug-sections",
"-f", debug_path,
"-F", real_debug_path,
path, NULL))
return FALSE;
}
}
return TRUE;
}
gboolean
builder_post_process (BuilderPostProcessFlags flags,
GFile *app_dir,
BuilderCache *cache,
BuilderContext *context,
GError **error)
{
g_autoptr(GPtrArray) changed = NULL;
if (!builder_cache_get_outstanding_changes (cache, &changed, error))
return FALSE;
if (flags & BUILDER_POST_PROCESS_FLAGS_PYTHON_TIMESTAMPS)
{
if (!builder_post_process_python_time_stamp (app_dir, changed,error))
return FALSE;
}
if (flags & BUILDER_POST_PROCESS_FLAGS_STRIP)
{
if (!builder_post_process_strip (app_dir, changed, error))
return FALSE;
}
else if (flags & BUILDER_POST_PROCESS_FLAGS_DEBUGINFO)
{
if (!builder_post_process_debuginfo (app_dir, changed, flags, context, error))
return FALSE;
}
return TRUE;
}
Reference in New Issue View Git Blame Copy Permalink