marko10_000
/
flatpak-builder
forked from Mirrors/flatpak-builder
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
flatpak-builder/src/builder-cache.c

1400 lines
42 KiB
C
Raw Permalink Blame History

/* builder-cache.c
*
* Copyright (C) 2015 Red Hat, Inc
*
* This file is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This file is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Authors:
* Alexander Larsson <alexl@redhat.com>
*/
#include "config.h"
#include <string.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/statfs.h>
#include <gio/gio.h>
#include <ostree.h>
#include "libglnx/libglnx.h"
#include "builder-flatpak-utils.h"
#include "builder-utils.h"
#include "builder-cache.h"
#include "builder-context.h"
struct BuilderCache
{
GObject parent;
BuilderContext *context;
GChecksum *checksum;
GFile *app_dir;
char *branch;
char *stage;
GHashTable *unused_stages;
char *last_parent;
char *current_checksum;
OstreeRepo *repo;
gboolean disabled;
OstreeRepoDevInoCache *devino_to_csum_cache;
};
typedef struct
{
GObjectClass parent_class;
} BuilderCacheClass;
G_DEFINE_TYPE (BuilderCache, builder_cache, G_TYPE_OBJECT);
enum {
PROP_0,
PROP_CONTEXT,
PROP_APP_DIR,
PROP_BRANCH,
LAST_PROP
};
#define OSTREE_GIO_FAST_QUERYINFO ("standard::name,standard::type,standard::size,standard::is-symlink,standard::symlink-target," \
"unix::device,unix::inode,unix::mode,unix::uid,unix::gid,unix::rdev")
static GPtrArray *builder_cache_get_changes_to (BuilderCache *self,
GFile *current_root,
GPtrArray **removals,
GError **error);
static void
builder_cache_finalize (GObject *object)
{
BuilderCache *self = (BuilderCache *) object;
g_clear_object (&self->context);
g_clear_object (&self->app_dir);
g_clear_object (&self->repo);
g_checksum_free (self->checksum);
g_free (self->branch);
g_free (self->last_parent);
g_free (self->stage);
g_free (self->current_checksum);
if (self->unused_stages)
g_hash_table_unref (self->unused_stages);
if (self->devino_to_csum_cache)
ostree_repo_devino_cache_unref (self->devino_to_csum_cache);
G_OBJECT_CLASS (builder_cache_parent_class)->finalize (object);
}
static void
builder_cache_get_property (GObject *object,
guint prop_id,
GValue *value,
GParamSpec *pspec)
{
BuilderCache *self = BUILDER_CACHE (object);
switch (prop_id)
{
case PROP_CONTEXT:
g_value_set_object (value, self->context);
break;
case PROP_APP_DIR:
g_value_set_object (value, self->app_dir);
break;
case PROP_BRANCH:
g_value_set_string (value, self->branch);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
}
}
static void
builder_cache_set_property (GObject *object,
guint prop_id,
const GValue *value,
GParamSpec *pspec)
{
BuilderCache *self = BUILDER_CACHE (object);
switch (prop_id)
{
case PROP_BRANCH:
g_free (self->branch);
self->branch = g_value_dup_string (value);
break;
case PROP_CONTEXT:
g_set_object (&self->context, g_value_get_object (value));
break;
case PROP_APP_DIR:
g_set_object (&self->app_dir, g_value_get_object (value));
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
}
}
static void
builder_cache_class_init (BuilderCacheClass *klass)
{
GObjectClass *object_class = G_OBJECT_CLASS (klass);
object_class->finalize = builder_cache_finalize;
object_class->get_property = builder_cache_get_property;
object_class->set_property = builder_cache_set_property;
g_object_class_install_property (object_class,
PROP_CONTEXT,
g_param_spec_object ("context",
"",
"",
BUILDER_TYPE_CONTEXT,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
g_object_class_install_property (object_class,
PROP_APP_DIR,
g_param_spec_object ("app-dir",
"",
"",
G_TYPE_FILE,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
g_object_class_install_property (object_class,
PROP_BRANCH,
g_param_spec_string ("branch",
"",
"",
NULL,
G_PARAM_READWRITE));
}
static void
builder_cache_init (BuilderCache *self)
{
self->checksum = g_checksum_new (G_CHECKSUM_SHA256);
self->devino_to_csum_cache = ostree_repo_devino_cache_new ();
}
BuilderCache *
builder_cache_new (BuilderContext *context,
GFile *app_dir,
const char *branch)
{
return g_object_new (BUILDER_TYPE_CACHE,
"context", context,
"app-dir", app_dir,
"branch", branch,
NULL);
}
GChecksum *
builder_cache_get_checksum (BuilderCache *self)
{
return self->checksum;
}
static void
append_escaped_stage (GString *s,
const char *stage)
{
while (*stage)
{
char c = *stage++;
if (g_ascii_isalnum (c) ||
c == '-' ||
c == '_' ||
c == '.')
g_string_append_c (s, c);
else
g_string_append_printf (s, "%x", c);
}
}
static char *
get_ref (BuilderCache *self, const char *stage)
{
GString *s = g_string_new (self->branch);
g_string_append_c (s, '/');
append_escaped_stage (s, stage);
return g_string_free (s, FALSE);
}
gboolean
builder_cache_open (BuilderCache *self,
GError **error)
{
g_autoptr(GKeyFile) config = NULL;
g_autofree char *old_mfsp = NULL;
self->repo = ostree_repo_new (builder_context_get_cache_dir (self->context));
if (!g_file_query_exists (builder_context_get_cache_dir (self->context), NULL))
{
g_autoptr(GFile) parent = g_file_get_parent (builder_context_get_cache_dir (self->context));
if (!flatpak_mkdir_p (parent, NULL, error))
return FALSE;
if (!ostree_repo_create (self->repo, OSTREE_REPO_MODE_BARE_USER_ONLY, NULL, error))
return FALSE;
}
if (!ostree_repo_open (self->repo, NULL, error))
return FALSE;
config = ostree_repo_copy_config (self->repo);
old_mfsp = g_key_file_get_value (config, "core", "min-free-space-percent", NULL);
if (g_strcmp0 (old_mfsp, "0") != 0)
{
g_key_file_set_value (config, "core", "min-free-space-percent", "0");
if (!ostree_repo_write_config (self->repo, config, error))
return FALSE;
/* Re-open */
g_clear_object (&self->repo);
self->repo = ostree_repo_new (builder_context_get_cache_dir (self->context));
if (!ostree_repo_open (self->repo, NULL, error))
return FALSE;
}
/* We don't need fsync on checkouts as they are transient, and we
rely on the syncfs() in the transaction commit for commits. */
ostree_repo_set_disable_fsync (self->repo, TRUE);
/* At one point we used just the branch name as a ref, make sure to
* remove this to handle using the branch as a subdir */
ostree_repo_set_ref_immediate (self->repo,
NULL,
self->branch,
NULL,
NULL, NULL);
/* List all stages first so we can purge unused ones at the end */
if (!ostree_repo_list_refs (self->repo,
self->branch,
&self->unused_stages,
NULL, error))
return FALSE;
return TRUE;
}
static gboolean
builder_cache_checkout (BuilderCache *self, const char *commit, gboolean delete_dir, GError **error)
{
g_autoptr(GError) my_error = NULL;
OstreeRepoCheckoutAtOptions options = { 0, };
if (delete_dir)
{
if (!g_file_delete (self->app_dir, NULL, &my_error) &&
!g_error_matches (my_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
g_propagate_error (error, g_steal_pointer (&my_error));
return FALSE;
}
if (!flatpak_mkdir_p (self->app_dir, NULL, error))
return FALSE;
}
/* If rofiles-fuse is disabled, we check out with force_copy
because we want to force the checkout to not use
hardlinks. Hard links into the cache without rofiles-fuse are notx
safe, as the build could mutate the cache. */
if (!builder_context_get_use_rofiles (self->context))
options.force_copy = TRUE;
options.mode = OSTREE_REPO_CHECKOUT_MODE_USER;
options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES;
options.devino_to_csum_cache = self->devino_to_csum_cache;
if (!ostree_repo_checkout_at (self->repo, &options,
AT_FDCWD, flatpak_file_get_path_cached (self->app_dir),
commit, NULL, error))
return FALSE;
/* There is a bug in ostree (https://github.com/ostreedev/ostree/issues/326) that
causes it to not reset mtime to 0 in them force_copy case. So we do that
manually */
if (options.force_copy &&
!flatpak_zero_mtime (AT_FDCWD, flatpak_file_get_path_cached (self->app_dir),
NULL, error))
return FALSE;
return TRUE;
}
gboolean
builder_cache_has_checkout (BuilderCache *self)
{
return self->disabled;
}
void
builder_cache_ensure_checkout (BuilderCache *self)
{
if (builder_cache_has_checkout (self))
return;
if (self->last_parent)
{
g_autoptr(GError) error = NULL;
g_print ("Everything cached, checking out from cache\n");
if (!builder_cache_checkout (self, self->last_parent, TRUE, &error))
g_error ("Failed to check out cache: %s", error->message);
}
self->disabled = TRUE;
}
static char *
builder_cache_get_current_ref (BuilderCache *self)
{
return get_ref (self, self->stage);
}
gboolean
builder_cache_lookup (BuilderCache *self,
const char *stage)
{
g_autofree char *commit = NULL;
g_autofree char *ref = NULL;
g_autoptr(GString) s = g_string_new ("");
g_free (self->stage);
self->stage = g_strdup (stage);
append_escaped_stage (s, stage);
g_hash_table_remove (self->unused_stages, s->str);
g_free (self->current_checksum);
self->current_checksum = g_strdup (g_checksum_get_string (self->checksum));
/* Reset the checksum, but feed it previous checksum so we chain it */
g_checksum_reset (self->checksum);
builder_cache_checksum_str (self, self->current_checksum);
if (self->disabled)
return FALSE;
ref = builder_cache_get_current_ref (self);
if (!ostree_repo_resolve_rev (self->repo, ref, TRUE, &commit, NULL))
goto checkout;
if (commit != NULL)
{
g_autoptr(GVariant) variant = NULL;
const gchar *subject;
if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_COMMIT, commit,
&variant, NULL))
goto checkout;
g_variant_get (variant, "(a{sv}aya(say)&s&stayay)", NULL, NULL, NULL,
&subject, NULL, NULL, NULL, NULL);
if (strcmp (subject, self->current_checksum) == 0)
{
g_free (self->last_parent);
self->last_parent = g_steal_pointer (&commit);
return TRUE;
}
}
checkout:
if (self->last_parent)
{
g_autoptr(GError) error = NULL;
g_print ("Cache miss, checking out last cache hit\n");
if (!builder_cache_checkout (self, self->last_parent, TRUE, &error))
g_error ("Failed to check out cache: %s", error->message);
}
self->disabled = TRUE; /* Don't use cache any more after first miss */
return FALSE;
}
static gboolean
mtree_empty (OstreeMutableTree *mtree)
{
GHashTable *files = ostree_mutable_tree_get_files (mtree);
GHashTable *subdirs = ostree_mutable_tree_get_subdirs (mtree);
return
g_hash_table_size (files) == 0 &&
g_hash_table_size (subdirs) == 0;
}
/* This takes a mutable tree and an existing OstreeRepoFile, and recursively
* removes all mtree files that already exists in the OstreeRepoFile.
* This is very useful to create a commit with just the new files, which
* we can then check out in order to get a the new hardlinks to the
* cache repo.
*/
static gboolean
mtree_prune_old_files (OstreeMutableTree *mtree,
OstreeRepoFile *old,
GError **error)
{
GHashTable *files = ostree_mutable_tree_get_files (mtree);
GHashTable *subdirs = ostree_mutable_tree_get_subdirs (mtree);
GHashTableIter iter;
gpointer key, value;
ostree_mutable_tree_set_contents_checksum (mtree, NULL);
if (old != NULL && !ostree_repo_file_ensure_resolved (old, error))
return FALSE;
g_hash_table_iter_init (&iter, files);
while (g_hash_table_iter_next (&iter, &key, &value))
{
const char *name = key;
const char *csum = value;
int n = -1;
gboolean is_dir;
g_autoptr(GVariant) container = NULL;
gboolean same = FALSE;
if (old)
n = ostree_repo_file_tree_find_child (old, name, &is_dir, &container);
if (n >= 0)
{
if (!is_dir)
{
g_autoptr(GVariant) old_csum_bytes = NULL;
g_autofree char *old_csum = NULL;
g_variant_get_child (container, n,
"(@s@ay)", NULL, &old_csum_bytes);
old_csum = ostree_checksum_from_bytes_v (old_csum_bytes);
if (strcmp (old_csum, csum) == 0)
same = TRUE; /* Modified file */
}
}
if (same)
g_hash_table_iter_remove (&iter);
}
g_hash_table_iter_init (&iter, subdirs);
while (g_hash_table_iter_next (&iter, &key, &value))
{
const char *name = key;
OstreeMutableTree *subdir = value;
g_autoptr(GFile) old_subdir = NULL;
int n = -1;
gboolean is_dir;
if (old)
n = ostree_repo_file_tree_find_child (old, name, &is_dir, NULL);
if (n >= 0 && is_dir)
old_subdir = g_file_get_child (G_FILE (old), name);
if (!mtree_prune_old_files (subdir, OSTREE_REPO_FILE (old_subdir), error))
return FALSE;
if (mtree_empty (subdir))
g_hash_table_iter_remove (&iter);
}
return TRUE;
}
static OstreeRepoCommitFilterResult
commit_filter (OstreeRepo *repo,
const char *path,
GFileInfo *file_info,
gpointer commit_data)
{
guint mode;
/* No user info */
g_file_info_set_attribute_uint32 (file_info, "unix::uid", 0);
g_file_info_set_attribute_uint32 (file_info, "unix::gid", 0);
/* In flatpak, there is no real reason for files to have different
* permissions based on the group or user really, everything is
* always used readonly for everyone. Having things be writeable
* for anyone but the user just causes risks for the system-installed
* case. So, we canonicalize the mode to writable only by the user,
* readable to all, and executable for all for directories and
* files that the user can execute.
*/
mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
mode = 0755 | S_IFDIR;
else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
{
/* If use can execute, make executable by all */
if (mode & S_IXUSR)
mode = 0755 | S_IFREG;
else /* otherwise executable by none */
mode = 0644 | S_IFREG;
}
g_file_info_set_attribute_uint32 (file_info, "unix::mode", mode);
return OSTREE_REPO_COMMIT_FILTER_ALLOW;
}
gboolean
builder_cache_commit (BuilderCache *self,
const char *body,
GError **error)
{
const char *current = NULL;
OstreeRepoCommitModifier *modifier = NULL;
g_autoptr(OstreeMutableTree) mtree = NULL;
g_autoptr(GFile) root = NULL;
g_autofree char *commit_checksum = NULL;
g_autofree char *new_commit_checksum = NULL;
gboolean res = FALSE;
g_autofree char *ref = NULL;
g_autoptr(GFile) last_root = NULL;
g_autoptr(GFile) new_root = NULL;
g_autoptr(GPtrArray) changes = NULL;
g_autoptr(GPtrArray) removals = NULL;
g_autoptr(GVariantDict) metadata_dict = NULL;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) changesv = NULL;
g_autoptr(GVariant) removalsv = NULL;
g_autoptr(GVariant) changesvz = NULL;
g_autoptr(GVariant) removalsvz = NULL;
g_print ("Committing stage %s to cache\n", self->stage);
/* We set all mtimes to 0 during a commit, to simulate what would happen when
running via flatpak deploy (and also if we checked out from the cache). */
if (!flatpak_zero_mtime (AT_FDCWD, flatpak_file_get_path_cached (self->app_dir),
NULL, NULL))
return FALSE;
if (!ostree_repo_prepare_transaction (self->repo, NULL, NULL, error))
return FALSE;
mtree = ostree_mutable_tree_new ();
modifier = ostree_repo_commit_modifier_new (OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS,
(OstreeRepoCommitFilter) commit_filter, NULL, NULL);
if (self->devino_to_csum_cache)
ostree_repo_commit_modifier_set_devino_cache (modifier, self->devino_to_csum_cache);
if (!ostree_repo_write_directory_to_mtree (self->repo, self->app_dir,
mtree, modifier, NULL, error))
goto out;
if (!ostree_repo_write_mtree (self->repo, mtree, &root, NULL, error))
goto out;
changes = builder_cache_get_changes_to (self, root, &removals, NULL);
metadata_dict = g_variant_dict_new (NULL);
changesv = g_variant_ref_sink (g_variant_new_strv ((const gchar * const *) changes->pdata, changes->len));
changesvz = flatpak_variant_compress (changesv);
g_variant_dict_insert_value (metadata_dict, "changesz", changesvz);
removalsv = g_variant_ref_sink (g_variant_new_strv ((const gchar * const *) removals->pdata, removals->len));
removalsvz = flatpak_variant_compress (removalsv);
g_variant_dict_insert_value (metadata_dict, "removalsz", removalsvz);
metadata = g_variant_ref_sink (g_variant_dict_end (metadata_dict));
current = self->current_checksum;
if (!ostree_repo_write_commit (self->repo, self->last_parent, current, body, metadata,
OSTREE_REPO_FILE (root),
&commit_checksum, NULL, error))
goto out;
ref = builder_cache_get_current_ref (self);
ostree_repo_transaction_set_ref (self->repo, NULL, ref, commit_checksum);
if (self->last_parent &&
!ostree_repo_read_commit (self->repo, self->last_parent, &last_root, NULL, NULL, error))
goto out;
if (!mtree_prune_old_files (mtree, OSTREE_REPO_FILE (last_root), error))
goto out;
if (!ostree_repo_write_mtree (self->repo, mtree, &new_root, NULL, error))
goto out;
if (!ostree_repo_write_commit (self->repo, NULL, current, body, metadata,
OSTREE_REPO_FILE (new_root),
&new_commit_checksum, NULL, error))
goto out;
if (!ostree_repo_commit_transaction (self->repo, NULL, NULL, error))
goto out;
/* Check out the just commited cache so we hardlinks to the cache */
if (builder_context_get_use_rofiles (self->context) &&
!builder_cache_checkout (self, new_commit_checksum, FALSE, error))
goto out;
g_free (self->last_parent);
self->last_parent = g_steal_pointer (&commit_checksum);
res = TRUE;
out:
if (!res)
{
if (!ostree_repo_abort_transaction (self->repo, NULL, NULL))
g_warning ("failed to abort transaction");
}
if (modifier)
ostree_repo_commit_modifier_unref (modifier);
return res;
}
typedef struct {
dev_t dev;
ino_t ino;
char checksum[OSTREE_SHA256_STRING_LEN+1];
} OstreeDevIno;
static const char *
devino_cache_lookup (OstreeRepoDevInoCache *devino_to_csum_cache,
guint32 device,
guint32 inode)
{
OstreeDevIno dev_ino_key;
OstreeDevIno *dev_ino_val;
GHashTable *cache = (GHashTable *)devino_to_csum_cache;
if (devino_to_csum_cache == NULL)
return NULL;
dev_ino_key.dev = device;
dev_ino_key.ino = inode;
dev_ino_val = g_hash_table_lookup (cache, &dev_ino_key);
if (!dev_ino_val)
return NULL;
return dev_ino_val->checksum;
}
static gboolean
get_file_checksum (OstreeRepoDevInoCache *devino_to_csum_cache,
GFile *f,
GFileInfo *f_info,
char **out_checksum,
GCancellable *cancellable,
GError **error)
{
g_autofree char *ret_checksum = NULL;
g_autofree guchar *csum = NULL;
if (OSTREE_IS_REPO_FILE (f))
{
ret_checksum = g_strdup (ostree_repo_file_get_checksum ((OstreeRepoFile*)f));
}
else
{
const char *cached = devino_cache_lookup (devino_to_csum_cache,
g_file_info_get_attribute_uint32 (f_info, "unix::device"),
g_file_info_get_attribute_uint64 (f_info, "unix::inode"));
if (cached)
ret_checksum = g_strdup (cached);
else
{
g_autoptr(GInputStream) in = NULL;
if (g_file_info_get_file_type (f_info) == G_FILE_TYPE_REGULAR)
{
in = (GInputStream*)g_file_read (f, cancellable, error);
if (!in)
return FALSE;
}
if (!ostree_checksum_file_from_input (f_info, NULL, in,
OSTREE_OBJECT_TYPE_FILE,
&csum, cancellable, error))
return FALSE;
ret_checksum = ostree_checksum_from_bytes (csum);
}
}
*out_checksum = g_steal_pointer (&ret_checksum);
return TRUE;
}
static gboolean
diff_files (OstreeRepoDevInoCache *devino_to_csum_cache,
GFile *a,
GFileInfo *a_info,
GFile *b,
GFileInfo *b_info,
gboolean *was_changed,
GCancellable *cancellable,
GError **error)
{
g_autofree char *checksum_a = NULL;
g_autofree char *checksum_b = NULL;
if (!get_file_checksum (devino_to_csum_cache, a, a_info, &checksum_a, cancellable, error))
return FALSE;
if (!get_file_checksum (devino_to_csum_cache, b, b_info, &checksum_b, cancellable, error))
return FALSE;
*was_changed = strcmp (checksum_a, checksum_b) != 0;
return TRUE;
}
static gboolean
diff_add_dir_recurse (GFile *d,
GPtrArray *added,
GCancellable *cancellable,
GError **error)
{
GError *temp_error = NULL;
g_autoptr(GFileEnumerator) dir_enum = NULL;
g_autoptr(GFile) child = NULL;
g_autoptr(GFileInfo) child_info = NULL;
dir_enum = g_file_enumerate_children (d, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
error);
if (!dir_enum)
return FALSE;
while ((child_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL)
{
const char *name;
name = g_file_info_get_name (child_info);
g_clear_object (&child);
child = g_file_get_child (d, name);
g_ptr_array_add (added, g_object_ref (child));
if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
{
if (!diff_add_dir_recurse (child, added, cancellable, error))
return FALSE;
}
g_clear_object (&child_info);
}
if (temp_error != NULL)
{
g_propagate_error (error, temp_error);
return FALSE;
}
return TRUE;
}
static gboolean
diff_dirs (OstreeRepoDevInoCache *devino_to_csum_cache,
GFile *a,
GFile *b,
GPtrArray *changed,
GCancellable *cancellable,
GError **error)
{
GError *temp_error = NULL;
g_autoptr(GFileEnumerator) dir_enum = NULL;
g_autoptr(GFile) child_a = NULL;
g_autoptr(GFile) child_b = NULL;
g_autoptr(GFileInfo) child_a_info = NULL;
g_autoptr(GFileInfo) child_b_info = NULL;
if (a == NULL)
{
if (!diff_add_dir_recurse (b, changed, cancellable, error))
return FALSE;
return TRUE;
}
dir_enum = g_file_enumerate_children (a, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_enum)
return FALSE;
while ((child_a_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL)
{
const char *name;
GFileType child_a_type;
GFileType child_b_type;
name = g_file_info_get_name (child_a_info);
g_clear_object (&child_a);
child_a = g_file_get_child (a, name);
child_a_type = g_file_info_get_file_type (child_a_info);
g_clear_object (&child_b);
child_b = g_file_get_child (b, name);
g_clear_object (&child_b_info);
child_b_info = g_file_query_info (child_b, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
&temp_error);
if (!child_b_info)
{
if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
g_clear_error (&temp_error);
/* Removed, ignore */
}
else
{
g_propagate_error (error, temp_error);
return FALSE;
}
}
else
{
child_b_type = g_file_info_get_file_type (child_b_info);
if (child_a_type != child_b_type)
{
g_ptr_array_add (changed, g_object_ref (child_b));
}
else
{
gboolean was_changed = FALSE;
if (!diff_files (devino_to_csum_cache,
child_a, child_a_info,
child_b, child_b_info,
&was_changed,
cancellable, error))
return FALSE;
if (was_changed)
g_ptr_array_add (changed, g_object_ref (child_b));
if (child_a_type == G_FILE_TYPE_DIRECTORY)
{
if (!diff_dirs (devino_to_csum_cache, child_a, child_b, changed,
cancellable, error))
return FALSE;
}
}
}
g_clear_object (&child_a_info);
}
if (temp_error != NULL)
{
g_propagate_error (error, temp_error);
return FALSE;
}
g_clear_object (&dir_enum);
dir_enum = g_file_enumerate_children (b, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_enum)
return FALSE;
g_clear_object (&child_b_info);
while ((child_b_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL)
{
const char *name;
name = g_file_info_get_name (child_b_info);
g_clear_object (&child_a);
child_a = g_file_get_child (a, name);
g_clear_object (&child_b);
child_b = g_file_get_child (b, name);
g_clear_object (&child_a_info);
child_a_info = g_file_query_info (child_a, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
&temp_error);
if (!child_a_info)
{
if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
g_clear_error (&temp_error);
g_ptr_array_add (changed, g_object_ref (child_b));
if (g_file_info_get_file_type (child_b_info) == G_FILE_TYPE_DIRECTORY)
{
if (!diff_add_dir_recurse (child_b, changed, cancellable, error))
return FALSE;
}
}
else
{
g_propagate_error (error, temp_error);
return FALSE;
}
}
g_clear_object (&child_b_info);
}
if (temp_error != NULL)
{
g_propagate_error (error, temp_error);
return FALSE;
}
return TRUE;
}
gboolean
builder_cache_get_outstanding_changes (BuilderCache *self,
GPtrArray **changed_out,
GError **error)
{
g_autoptr(GPtrArray) changed = g_ptr_array_new_with_free_func (g_object_unref);
g_autoptr(GPtrArray) changed_paths = g_ptr_array_new_with_free_func (g_free);
g_autoptr(GFile) last_root = NULL;
int i;
if (self->last_parent &&
!ostree_repo_read_commit (self->repo, self->last_parent, &last_root, NULL, NULL, error))
return FALSE;
if (!diff_dirs (self->devino_to_csum_cache,
last_root,
self->app_dir,
changed,
NULL, error))
return FALSE;
for (i = 0; i < changed->len; i++)
{
GFile *changed_file = g_ptr_array_index (changed, i);
char *path = g_file_get_relative_path (self->app_dir, changed_file);
g_ptr_array_add (changed_paths, path);
}
if (changed_out)
*changed_out = g_steal_pointer (&changed_paths);
return TRUE;
}
static int
cmpstringp (const void *p1, const void *p2)
{
return strcmp (*(char * const *) p1, *(char * const *) p2);
}
static GPtrArray *
get_changes (BuilderCache *self,
GFile *from,
GFile *to,
GPtrArray **removed_out,
GError **error)
{
g_autoptr(GPtrArray) added = g_ptr_array_new_with_free_func (g_object_unref);
g_autoptr(GPtrArray) modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref);
g_autoptr(GPtrArray) removed = g_ptr_array_new_with_free_func (g_object_unref);
g_autoptr(GPtrArray) changed_paths = g_ptr_array_new_with_free_func (g_free);
int i;
if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE,
from,
to,
modified,
removed,
added,
NULL, error))
return NULL;
for (i = 0; i < added->len; i++)
{
char *path = g_file_get_relative_path (to, g_ptr_array_index (added, i));
g_ptr_array_add (changed_paths, path);
}
for (i = 0; i < modified->len; i++)
{
OstreeDiffItem *modified_item = g_ptr_array_index (modified, i);
char *path = g_file_get_relative_path (to, modified_item->target);
g_ptr_array_add (changed_paths, path);
}
g_ptr_array_sort (changed_paths, cmpstringp);
if (removed_out)
{
GPtrArray *removed_paths = g_ptr_array_new_with_free_func (g_free);
for (i = 0; i < removed->len; i++)
{
char *path = g_file_get_relative_path (to, g_ptr_array_index (removed, i));
g_ptr_array_add (removed_paths, path);
}
*removed_out = removed_paths;
}
return g_steal_pointer (&changed_paths);
}
/* This returns removals too */
static GPtrArray *
get_all_changes (BuilderCache *self,
GFile *from,
GFile *to,
GError **error)
{
g_autoptr(GPtrArray) added = g_ptr_array_new_with_free_func (g_object_unref);
g_autoptr(GPtrArray) modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref);
g_autoptr(GPtrArray) removed = g_ptr_array_new_with_free_func (g_object_unref);
g_autoptr(GPtrArray) changed_paths = g_ptr_array_new_with_free_func (g_free);
int i;
if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE,
from,
to,
modified,
removed,
added,
NULL, error))
return NULL;
for (i = 0; i < added->len; i++)
{
char *path = g_file_get_relative_path (to, g_ptr_array_index (added, i));
g_ptr_array_add (changed_paths, path);
}
for (i = 0; i < modified->len; i++)
{
OstreeDiffItem *modified_item = g_ptr_array_index (modified, i);
char *path = g_file_get_relative_path (to, modified_item->target);
g_ptr_array_add (changed_paths, path);
}
for (i = 0; i < removed->len; i++)
{
char *path = g_file_get_relative_path (to, g_ptr_array_index (removed, i));
g_ptr_array_add (changed_paths, path);
}
g_ptr_array_sort (changed_paths, cmpstringp);
return g_steal_pointer (&changed_paths);
}
/* This returns removals too */
GPtrArray *
builder_cache_get_all_changes (BuilderCache *self,
GError **error)
{
g_autoptr(GFile) init_root = NULL;
g_autoptr(GFile) finish_root = NULL;
g_autofree char *init_commit = NULL;
g_autofree char *finish_commit = NULL;
g_autofree char *init_ref = get_ref (self, "init");
g_autofree char *finish_ref = get_ref (self, "finish");
if (!ostree_repo_resolve_rev (self->repo, init_ref, FALSE, &init_commit, NULL))
return FALSE;
if (!ostree_repo_resolve_rev (self->repo, finish_ref, FALSE, &finish_commit, NULL))
return FALSE;
if (!ostree_repo_read_commit (self->repo, init_commit, &init_root, NULL, NULL, error))
return NULL;
if (!ostree_repo_read_commit (self->repo, finish_commit, &finish_root, NULL, NULL, error))
return NULL;
return get_all_changes (self, init_root, finish_root, error);
}
static GPtrArray *
builder_cache_get_changes_to (BuilderCache *self,
GFile *current_root,
GPtrArray **removals,
GError **error)
{
g_autoptr(GFile) parent_root = NULL;
if (self->last_parent != NULL &&
!ostree_repo_read_commit (self->repo, self->last_parent, &parent_root, NULL, NULL, error))
return FALSE;
return get_changes (self, parent_root, current_root, removals, error);
}
GPtrArray *
builder_cache_get_changes (BuilderCache *self,
GError **error)
{
g_autoptr(GFile) current_root = NULL;
g_autoptr(GFile) parent_root = NULL;
g_autoptr(GVariant) variant = NULL;
g_autoptr(GVariant) commit_metadata = NULL;
g_autofree char *parent_commit = NULL;
g_autoptr(GVariant) changesz_v = NULL;
g_autoptr(GVariant) changes_v = NULL;
if (!ostree_repo_read_commit (self->repo, self->last_parent, &current_root, NULL, NULL, error))
return NULL;
if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_COMMIT, self->last_parent,
&variant, NULL))
return NULL;
commit_metadata = g_variant_get_child_value (variant, 0);
changesz_v = g_variant_lookup_value (commit_metadata, "changesz", G_VARIANT_TYPE_BYTESTRING);
if (changesz_v)
changes_v = flatpak_variant_uncompress (changesz_v, G_VARIANT_TYPE ("as"));
else
changes_v = g_variant_lookup_value (commit_metadata, "changes", G_VARIANT_TYPE ("as"));
if (changes_v)
{
g_autoptr(GPtrArray) changed_paths = g_ptr_array_new_with_free_func (g_free);
int i;
for (i = 0; i < g_variant_n_children (changes_v); i++)
{
char *str;
g_variant_get_child (changes_v, i, "s", &str);
g_ptr_array_add (changed_paths, str);
}
return g_steal_pointer (&changed_paths);
}
parent_commit = ostree_commit_get_parent (variant);
if (parent_commit != NULL)
{
if (!ostree_repo_read_commit (self->repo, parent_commit, &parent_root, NULL, NULL, error))
return FALSE;
}
return get_changes (self, parent_root, current_root, NULL, error);
}
GPtrArray *
builder_cache_get_files (BuilderCache *self,
GError **error)
{
g_autoptr(GFile) current_root = NULL;
if (!ostree_repo_read_commit (self->repo, self->last_parent, &current_root, NULL, NULL, error))
return NULL;
return get_changes (self, NULL, current_root, NULL, error);
}
void
builder_cache_disable_lookups (BuilderCache *self)
{
self->disabled = TRUE;
}
gboolean
builder_gc (BuilderCache *self,
gboolean prune_unused_stages,
GError **error)
{
gint objects_total;
gint objects_pruned;
guint64 pruned_object_size_total;
GHashTableIter iter;
gpointer key, value;
if (prune_unused_stages)
{
g_hash_table_iter_init (&iter, self->unused_stages);
while (g_hash_table_iter_next (&iter, &key, &value))
{
const char *unused_stage = (const char *) key;
g_autofree char *unused_ref = get_ref (self, unused_stage);
g_debug ("Removing unused ref %s", unused_ref);
if (!ostree_repo_set_ref_immediate (self->repo,
NULL,
unused_ref,
NULL,
NULL, error))
return FALSE;
}
}
g_print ("Pruning cache\n");
return ostree_repo_prune (self->repo,
OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, -1,
&objects_total,
&objects_pruned,
&pruned_object_size_total,
NULL, error);
}
/* Only add to cache if non-empty. This means we can add
these things compatibly without invalidating the cache.
This is useful if empty means no change from what was
before */
void
builder_cache_checksum_compat_str (BuilderCache *self,
const char *str)
{
if (str)
builder_cache_checksum_str (self, str);
}
void
builder_cache_checksum_str (BuilderCache *self,
const char *str)
{
/* We include the terminating zero so that we make
* a difference between NULL and "". */
if (str)
g_checksum_update (self->checksum, (const guchar *) str, strlen (str) + 1);
else
/* Always add something so we can't be fooled by a sequence like
NULL, "a" turning into "a", NULL. */
g_checksum_update (self->checksum, (const guchar *) "\1", 1);
}
/* Only add to cache if non-empty. This means we can add
these things compatibly without invalidating the cache.
This is useful if empty means no change from what was
before */
void
builder_cache_checksum_compat_strv (BuilderCache *self,
char **strv)
{
if (strv != NULL && strv[0] != NULL)
builder_cache_checksum_strv (self, strv);
}
void
builder_cache_checksum_strv (BuilderCache *self,
char **strv)
{
int i;
if (strv)
{
g_checksum_update (self->checksum, (const guchar *) "\1", 1);
for (i = 0; strv[i] != NULL; i++)
builder_cache_checksum_str (self, strv[i]);
}
else
{
g_checksum_update (self->checksum, (const guchar *) "\2", 1);
}
}
void
builder_cache_checksum_boolean (BuilderCache *self,
gboolean val)
{
if (val)
g_checksum_update (self->checksum, (const guchar *) "\1", 1);
else
g_checksum_update (self->checksum, (const guchar *) "\0", 1);
}
/* Only add to cache if true. This means we can add
these things compatibly without invalidating the cache.
This is useful if false means no change from what was
before */
void
builder_cache_checksum_compat_boolean (BuilderCache *self,
gboolean val)
{
if (val)
builder_cache_checksum_boolean (self, val);
}
void
builder_cache_checksum_uint32 (BuilderCache *self,
guint32 val)
{
guchar v[4];
v[0] = (val >> 0) & 0xff;
v[1] = (val >> 8) & 0xff;
v[2] = (val >> 16) & 0xff;
v[3] = (val >> 24) & 0xff;
g_checksum_update (self->checksum, v, 4);
}
void
builder_cache_checksum_random (BuilderCache *self)
{
builder_cache_checksum_uint32 (self, g_random_int ());
builder_cache_checksum_uint32 (self, g_random_int ());
}
void
builder_cache_checksum_uint64 (BuilderCache *self,
guint64 val)
{
guchar v[8];
v[0] = (val >> 0) & 0xff;
v[1] = (val >> 8) & 0xff;
v[2] = (val >> 16) & 0xff;
v[3] = (val >> 24) & 0xff;
v[4] = (val >> 32) & 0xff;
v[5] = (val >> 40) & 0xff;
v[6] = (val >> 48) & 0xff;
v[7] = (val >> 56) & 0xff;
g_checksum_update (self->checksum, v, 8);
}
void
builder_cache_checksum_data (BuilderCache *self,
guint8 *data,
gsize len)
{
g_checksum_update (self->checksum, data, len);
}
Reference in New Issue View Git Blame Copy Permalink