flatpak-builder: bundle module sources as runtime

This adds a step to the build process to bundle
the module sources, used for building the flatpak,
as a runtime extension.

The sources can then be installed like the
debug or translation runtime.

This also adds an option to flatpak-builder
for specifying sources directories. One can specify
source dirctories with the use-sources argument. This
will skip the download part of the processing
and will extract the sources from the given sources
directory directly for further processing.

Those source directories do need the same
structure as the ones that flatpak-builder
creates during processing in .flatpak-builder
and which is also used in the exported sources
runtime.
tingping/wmclass
Simon Schampijer 2017-03-31 07:56:10 +02:00
parent b426306884
commit 894311e243
17 changed files with 353 additions and 0 deletions

View File

@ -47,6 +47,7 @@ struct BuilderContext
char *stop_at;
GFile *download_dir;
GPtrArray *sources_dirs;
GFile *state_dir;
GFile *build_dir;
GFile *cache_dir;
@ -108,6 +109,8 @@ builder_context_finalize (GObject *object)
g_strfreev (self->cleanup_platform);
glnx_release_lock_file(&self->rofiles_file_lock);
g_clear_pointer (&self->sources_dirs, g_ptr_array_unref);
G_OBJECT_CLASS (builder_context_parent_class)->finalize (object);
}
@ -248,6 +251,20 @@ builder_context_get_download_dir (BuilderContext *self)
return self->download_dir;
}
GPtrArray *
builder_context_get_sources_dirs (BuilderContext *self)
{
return self->sources_dirs;
}
void
builder_context_set_sources_dirs (BuilderContext *self,
GPtrArray *sources_dirs)
{
g_clear_pointer (&self->sources_dirs, g_ptr_array_unref);
self->sources_dirs = g_ptr_array_ref (sources_dirs);
}
GFile *
builder_context_get_cache_dir (BuilderContext *self)
{

View File

@ -49,6 +49,9 @@ GFile * builder_context_allocate_build_subdir (BuilderContext *self,
GError **error);
GFile * builder_context_get_ccache_dir (BuilderContext *self);
GFile * builder_context_get_download_dir (BuilderContext *self);
GPtrArray * builder_context_get_sources_dirs (BuilderContext *self);
void builder_context_set_sources_dirs (BuilderContext *self,
GPtrArray *sources_dirs);
SoupSession * builder_context_get_soup_session (BuilderContext *self);
const char * builder_context_get_arch (BuilderContext *self);
void builder_context_set_arch (BuilderContext *self,

View File

@ -56,6 +56,19 @@ git_get_mirror_dir (const char *url_or_path,
g_autoptr(GFile) git_dir = NULL;
g_autofree char *filename = NULL;
g_autofree char *git_dir_path = NULL;
GPtrArray *sources_dirs = NULL;
int i;
sources_dirs = builder_context_get_sources_dirs (context);
for (i = 0; sources_dirs != NULL && i < sources_dirs->len; i++)
{
GFile* sources_root = g_ptr_array_index (sources_dirs, i);
g_autoptr(GFile) local_git_dir = g_file_get_child (sources_root, "git");
g_autofree char *local_filename = builder_uri_to_filename (url_or_path);
g_autoptr(GFile) file = g_file_get_child (local_git_dir, local_filename);
if (g_file_query_exists (file, NULL))
return g_steal_pointer (&file);
}
git_dir = g_file_get_child (builder_context_get_state_dir (context),
"git");
@ -216,6 +229,7 @@ git_mirror_submodules (const char *repo_location,
gboolean
builder_git_mirror_repo (const char *repo_location,
const char *destination_path,
gboolean update,
gboolean mirror_submodules,
gboolean disable_fsck,
@ -228,6 +242,16 @@ builder_git_mirror_repo (const char *repo_location,
mirror_dir = git_get_mirror_dir (repo_location, context);
if (destination_path != NULL)
{
g_autofree char *file_name = g_file_get_basename (mirror_dir);
g_autofree char *destination_file_path = g_build_filename (destination_path,
file_name,
NULL);
g_object_unref (mirror_dir);
mirror_dir = g_file_new_for_path (destination_file_path);
}
if (!g_file_query_exists (mirror_dir, NULL))
{
g_autofree char *filename = g_file_get_basename (mirror_dir);

View File

@ -26,6 +26,7 @@
G_BEGIN_DECLS
gboolean builder_git_mirror_repo (const char *repo_location,
const char *destination_path,
gboolean update,
gboolean mirror_submodules,
gboolean disable_fsck,

View File

@ -62,6 +62,7 @@ static char *opt_subject;
static char *opt_body;
static char *opt_gpg_homedir;
static char **opt_key_ids;
static char **opt_sources_dirs;
static int opt_jobs;
static GOptionEntry entries[] = {
@ -76,6 +77,7 @@ static GOptionEntry entries[] = {
{ "disable-download", 0, 0, G_OPTION_ARG_NONE, &opt_disable_download, "Don't download any new sources", NULL },
{ "disable-updates", 0, 0, G_OPTION_ARG_NONE, &opt_disable_updates, "Only download missing sources, never update to latest vcs version", NULL },
{ "download-only", 0, 0, G_OPTION_ARG_NONE, &opt_download_only, "Only download sources, don't build", NULL },
{ "use-sources", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_sources_dirs, "Build the sources specified by SOURCE-DIR, multiple uses of this option possible", "SOURCE-DIR"},
{ "build-only", 0, 0, G_OPTION_ARG_NONE, &opt_build_only, "Stop after build, don't run clean and finish phases", NULL },
{ "finish-only", 0, 0, G_OPTION_ARG_NONE, &opt_finish_only, "Only run clean and finish and export phases", NULL },
{ "allow-missing-runtimes", 0, 0, G_OPTION_ARG_NONE, &opt_allow_missing_runtimes, "Don't fail if runtime and sdk missing", NULL },
@ -322,6 +324,18 @@ main (int argc,
builder_context_set_jobs (build_context, opt_jobs);
builder_context_set_rebuild_on_sdk_change (build_context, opt_rebuild_on_sdk_change);
if (opt_sources_dirs)
{
g_autoptr(GPtrArray) sources_dirs = NULL;
sources_dirs = g_ptr_array_new_with_free_func (g_object_unref);
for (i = 0; opt_sources_dirs != NULL && opt_sources_dirs[i] != NULL; i++)
{
GFile *file = g_file_new_for_commandline_arg (opt_sources_dirs[i]);
g_ptr_array_add (sources_dirs, file);
}
builder_context_set_sources_dirs (build_context, sources_dirs);
}
if (opt_arch)
builder_context_set_arch (build_context, opt_arch);
@ -498,6 +512,7 @@ main (int argc,
if (!opt_finish_only &&
!opt_disable_download &&
!opt_sources_dirs &&
!builder_manifest_download (manifest, !opt_disable_updates, opt_build_shell, build_context, &error))
{
g_printerr ("Failed to download sources: %s\n", error->message);
@ -586,6 +601,7 @@ main (int argc,
if (!opt_build_only && opt_repo && builder_cache_has_checkout (cache))
{
g_autoptr(GFile) debuginfo_metadata = NULL;
g_autoptr(GFile) sourcesinfo_metadata = NULL;
g_print ("Exporting %s to repo\n", builder_manifest_get_id (manifest));
@ -648,6 +664,23 @@ main (int argc,
}
}
/* Export sources extensions */
sourcesinfo_metadata = g_file_get_child (app_dir, "metadata.sourcesinfo");
if (g_file_query_exists (sourcesinfo_metadata, NULL))
{
g_autofree char *sources_id = builder_manifest_get_sources_id (manifest);
g_print ("Exporting %s to repo\n", sources_id);
if (!do_export (build_context, &error, TRUE,
"--metadata=metadata.sourcesinfo",
builder_context_get_build_runtime (build_context) ? "--files=usr/sources" : "--files=sources",
opt_repo, app_dir_path, builder_manifest_get_branch (manifest, opt_default_branch), NULL))
{
g_printerr ("Export failed: %s\n", error->message);
return 1;
}
}
/* Export platform */
platform_id = builder_manifest_get_id_platform (manifest);
if (builder_context_get_build_runtime (build_context) &&

View File

@ -1044,6 +1044,13 @@ builder_manifest_get_debug_id (BuilderManifest *self)
return g_strdup_printf ("%s.Debug", id);
}
char *
builder_manifest_get_sources_id (BuilderManifest *self)
{
g_autofree char *id = make_valid_id_prefix (self->id);
return g_strdup_printf ("%s.Sources", id);
}
const char *
builder_manifest_get_id_platform (BuilderManifest *self)
{
@ -2051,6 +2058,7 @@ builder_manifest_finish (BuilderManifest *self,
{
g_autoptr(GFile) manifest_file = NULL;
g_autoptr(GFile) debuginfo_dir = NULL;
g_autoptr(GFile) sources_dir = NULL;
g_autoptr(GFile) locale_parent_dir = NULL;
g_autofree char *json = NULL;
g_autofree char *commandline = NULL;
@ -2185,11 +2193,13 @@ builder_manifest_finish (BuilderManifest *self,
{
debuginfo_dir = g_file_resolve_relative_path (app_dir, "usr/lib/debug");
locale_parent_dir = g_file_resolve_relative_path (app_dir, "usr/" LOCALES_SEPARATE_DIR);
sources_dir = g_file_resolve_relative_path (app_dir, "usr/sources");
}
else
{
debuginfo_dir = g_file_resolve_relative_path (app_dir, "files/lib/debug");
locale_parent_dir = g_file_resolve_relative_path (app_dir, "files/" LOCALES_SEPARATE_DIR);
sources_dir = g_file_resolve_relative_path (app_dir, "sources");
}
if (self->separate_locales && g_file_query_exists (locale_parent_dir, NULL))
@ -2264,6 +2274,21 @@ builder_manifest_finish (BuilderManifest *self,
return FALSE;
}
if (g_file_query_exists (sources_dir, NULL))
{
g_autoptr(GFile) metadata_sources_file = NULL;
g_autofree char *metadata_contents = NULL;
g_autofree char *sources_id = builder_manifest_get_sources_id (self);
metadata_sources_file = g_file_get_child (app_dir, "metadata.sources");
metadata_contents = g_strdup_printf ("[Runtime]\n"
"name=%s\n", sources_id);
if (!g_file_set_contents (flatpak_file_get_path_cached (metadata_sources_file),
metadata_contents, strlen (metadata_contents), error))
return FALSE;
}
if (!builder_context_disable_rofiles (context, error))
return FALSE;

View File

@ -49,6 +49,7 @@ void builder_manifest_set_demarshal_buid_context (BuilderContext *build_context)
const char * builder_manifest_get_id (BuilderManifest *self);
char * builder_manifest_get_locale_id (BuilderManifest *self);
char * builder_manifest_get_debug_id (BuilderManifest *self);
char * builder_manifest_get_sources_id (BuilderManifest *self);
const char * builder_manifest_get_id_platform (BuilderManifest *self);
char * builder_manifest_get_locale_id_platform (BuilderManifest *self);
BuilderOptions *builder_manifest_get_build_options (BuilderManifest *self);

View File

@ -882,6 +882,30 @@ builder_module_extract_sources (BuilderModule *self,
return TRUE;
}
gboolean
builder_module_bundle_sources (BuilderModule *self,
BuilderContext *context,
GError **error)
{
GList *l;
for (l = self->sources; l != NULL; l = l->next)
{
BuilderSource *source = l->data;
if (!builder_source_is_enabled (source, context))
continue;
if (!builder_source_bundle (source, context, error))
{
g_prefix_error (error, "module %s: ", self->name);
return FALSE;
}
}
return TRUE;
}
static GPtrArray *
setup_build_args (GFile *app_dir,
const char *module_name,
@ -1163,6 +1187,9 @@ builder_module_build (BuilderModule *self,
if (!builder_module_extract_sources (self, source_dir, context, error))
return FALSE;
if (!builder_module_bundle_sources (self, context, error))
return FALSE;
if (self->subdir != NULL && self->subdir[0] != 0)
{
source_subdir = g_file_resolve_relative_path (source_dir, self->subdir);

View File

@ -62,6 +62,9 @@ gboolean builder_module_extract_sources (BuilderModule *self,
GFile *dest,
BuilderContext *context,
GError **error);
gboolean builder_module_bundle_sources (BuilderModule *self,
BuilderContext *context,
GError **error);
gboolean builder_module_ensure_writable (BuilderModule *self,
BuilderCache *cache,
BuilderContext *context,

View File

@ -219,6 +219,8 @@ get_download_location (BuilderSourceArchive *self,
GFile *download_dir = NULL;
g_autoptr(GFile) sha256_dir = NULL;
g_autoptr(GFile) file = NULL;
GPtrArray *sources_dirs = NULL;
int i;
uri = get_uri (self, error);
if (uri == NULL)
@ -234,6 +236,17 @@ get_download_location (BuilderSourceArchive *self,
return FALSE;
}
sources_dirs = builder_context_get_sources_dirs (context);
for (i = 0; sources_dirs != NULL && i < sources_dirs->len; i++)
{
GFile* sources_root = g_ptr_array_index (sources_dirs, i);
g_autoptr(GFile) local_download_dir = g_file_get_child (sources_root, "downloads");
g_autoptr(GFile) local_sha256_dir = g_file_get_child (local_download_dir, self->sha256);
g_autoptr(GFile) local_file = g_file_get_child (local_sha256_dir, base_name);
if (g_file_query_exists (local_file, NULL))
return g_steal_pointer (&local_file);
}
download_dir = builder_context_get_download_dir (context);
sha256_dir = g_file_get_child (download_dir, self->sha256);
file = g_file_get_child (sha256_dir, base_name);
@ -580,6 +593,52 @@ builder_source_archive_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_archive_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
BuilderSourceArchive *self = BUILDER_SOURCE_ARCHIVE (source);
g_autoptr(GFile) file = NULL;
g_autoptr(GFile) download_dir = NULL;
g_autoptr(GFile) destination_file = NULL;
g_autofree char *download_dir_path = NULL;
g_autofree char *file_name = NULL;
g_autofree char *destination_file_path = NULL;
g_autofree char *app_dir_path = g_file_get_path (builder_context_get_app_dir (context));
gboolean is_local;
file = get_source_file (self, context, &is_local, error);
if (file == NULL)
return FALSE;
download_dir_path = g_build_filename (app_dir_path,
"sources",
"downloads",
self->sha256,
NULL);
download_dir = g_file_new_for_path (download_dir_path);
if (!g_file_query_exists (download_dir, NULL) &&
!g_file_make_directory_with_parents (download_dir, NULL, error))
return FALSE;
file_name = g_file_get_basename (file);
destination_file_path = g_build_filename (download_dir_path,
file_name,
NULL);
destination_file = g_file_new_for_path (destination_file_path);
if (!g_file_copy (file, destination_file,
G_FILE_COPY_OVERWRITE,
NULL,
NULL, NULL,
error))
return FALSE;
return TRUE;
}
static void
builder_source_archive_checksum (BuilderSource *source,
BuilderCache *cache,
@ -606,6 +665,7 @@ builder_source_archive_class_init (BuilderSourceArchiveClass *klass)
source_class->show_deps = builder_source_archive_show_deps;
source_class->download = builder_source_archive_download;
source_class->extract = builder_source_archive_extract;
source_class->bundle = builder_source_archive_bundle;
source_class->checksum = builder_source_archive_checksum;
g_object_class_install_property (object_class,

View File

@ -170,6 +170,8 @@ get_download_location (BuilderSourceFile *self,
GFile *download_dir = NULL;
g_autoptr(GFile) sha256_dir = NULL;
g_autoptr(GFile) file = NULL;
GPtrArray *sources_dirs = NULL;
int i;
uri = get_uri (self, error);
if (uri == NULL)
@ -191,6 +193,17 @@ get_download_location (BuilderSourceFile *self,
return FALSE;
}
sources_dirs = builder_context_get_sources_dirs (context);
for (i = 0; sources_dirs != NULL && i < sources_dirs->len; i++)
{
GFile* sources_root = g_ptr_array_index (sources_dirs, i);
g_autoptr(GFile) local_download_dir = g_file_get_child (sources_root, "downloads");
g_autoptr(GFile) local_sha256_dir = g_file_get_child (local_download_dir, self->sha256);
g_autoptr(GFile) local_file = g_file_get_child (local_sha256_dir, base_name);
if (g_file_query_exists (local_file, NULL))
return g_steal_pointer (&local_file);
}
download_dir = builder_context_get_download_dir (context);
sha256_dir = g_file_get_child (download_dir, self->sha256);
file = g_file_get_child (sha256_dir, base_name);
@ -423,6 +436,52 @@ builder_source_file_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_file_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
BuilderSourceFile *self = BUILDER_SOURCE_FILE (source);
g_autoptr(GFile) file = NULL;
g_autoptr(GFile) download_dir = NULL;
g_autoptr(GFile) destination_file = NULL;
g_autofree char *download_dir_path = NULL;
g_autofree char *file_name = NULL;
g_autofree char *destination_file_path = NULL;
g_autofree char *app_dir_path = g_file_get_path (builder_context_get_app_dir (context));
gboolean is_local, is_inline;
file = get_source_file (self, context, &is_local, &is_inline, error);
if (file == NULL)
return FALSE;
download_dir_path = g_build_filename (app_dir_path,
"sources",
"downloads",
self->sha256,
NULL);
download_dir = g_file_new_for_path (download_dir_path);
if (!g_file_query_exists (download_dir, NULL) &&
!g_file_make_directory_with_parents (download_dir, NULL, error))
return FALSE;
file_name = g_file_get_basename (file);
destination_file_path = g_build_filename (download_dir_path,
file_name,
NULL);
destination_file = g_file_new_for_path (destination_file_path);
if (!g_file_copy (file, destination_file,
G_FILE_COPY_OVERWRITE,
NULL,
NULL, NULL,
error))
return FALSE;
return TRUE;
}
static gboolean
builder_source_file_update (BuilderSource *source,
BuilderContext *context,
@ -498,6 +557,7 @@ builder_source_file_class_init (BuilderSourceFileClass *klass)
source_class->show_deps = builder_source_file_show_deps;
source_class->download = builder_source_file_download;
source_class->extract = builder_source_file_extract;
source_class->bundle = builder_source_file_bundle;
source_class->update = builder_source_file_update;
source_class->checksum = builder_source_file_checksum;

View File

@ -245,6 +245,43 @@ builder_source_git_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_git_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
BuilderSourceGit *self = BUILDER_SOURCE_GIT (source);
g_autofree char *location = NULL;
g_autoptr(GFile) mirror_dir = NULL;
g_autofree char *mirror_dir_path = NULL;
g_autofree char *app_dir_path = g_file_get_path (builder_context_get_app_dir (context));
location = get_url_or_path (self, context, error);
if (location == NULL)
return FALSE;
mirror_dir_path = g_build_filename (app_dir_path,
"sources",
"git",
NULL);
mirror_dir = g_file_new_for_path (mirror_dir_path);
if (!g_file_query_exists (mirror_dir, NULL) &&
!g_file_make_directory_with_parents (mirror_dir, NULL, error))
return FALSE;
if (!builder_git_mirror_repo (location,
mirror_dir_path,
FALSE, TRUE,
get_branch (self),
context,
error))
return FALSE;
return TRUE;
}
static void
builder_source_git_checksum (BuilderSource *source,
BuilderCache *cache,
@ -311,6 +348,7 @@ builder_source_git_class_init (BuilderSourceGitClass *klass)
source_class->download = builder_source_git_download;
source_class->extract = builder_source_git_extract;
source_class->bundle = builder_source_git_bundle;
source_class->update = builder_source_git_update;
source_class->checksum = builder_source_git_checksum;

View File

@ -254,6 +254,16 @@ builder_source_patch_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_patch_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
// the patches are part of the source archives
// or repositories, no need to bundle anything here
return TRUE;
}
static void
builder_source_patch_checksum (BuilderSource *source,
BuilderCache *cache,
@ -290,6 +300,7 @@ builder_source_patch_class_init (BuilderSourcePatchClass *klass)
source_class->show_deps = builder_source_patch_show_deps;
source_class->download = builder_source_patch_download;
source_class->extract = builder_source_patch_extract;
source_class->bundle = builder_source_patch_bundle;
source_class->checksum = builder_source_patch_checksum;
g_object_class_install_property (object_class,

View File

@ -174,6 +174,16 @@ builder_source_script_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_script_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
// no need to bundle anything here as this part
// can be reconstructed from the manifest
return TRUE;
}
static void
builder_source_script_checksum (BuilderSource *source,
BuilderCache *cache,
@ -197,6 +207,7 @@ builder_source_script_class_init (BuilderSourceScriptClass *klass)
source_class->download = builder_source_script_download;
source_class->extract = builder_source_script_extract;
source_class->bundle = builder_source_script_bundle;
source_class->checksum = builder_source_script_checksum;
g_object_class_install_property (object_class,

View File

@ -178,6 +178,16 @@ builder_source_shell_extract (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_shell_bundle (BuilderSource *source,
BuilderContext *context,
GError **error)
{
// no need to bundle anything here as this part
// can be reconstructed from the manifest
return TRUE;
}
static void
builder_source_shell_checksum (BuilderSource *source,
BuilderCache *cache,
@ -200,6 +210,7 @@ builder_source_shell_class_init (BuilderSourceShellClass *klass)
source_class->download = builder_source_shell_download;
source_class->extract = builder_source_shell_extract;
source_class->bundle = builder_source_shell_bundle;
source_class->checksum = builder_source_shell_checksum;
g_object_class_install_property (object_class,

View File

@ -150,6 +150,16 @@ builder_source_real_extract (BuilderSource *self,
return FALSE;
}
static gboolean
builder_source_real_bundle (BuilderSource *self,
BuilderContext *context,
GError **error)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Bundle not implemented for type %s", g_type_name_from_instance ((GTypeInstance *) self));
return FALSE;
}
static gboolean
builder_source_real_update (BuilderSource *self,
BuilderContext *context,
@ -170,6 +180,7 @@ builder_source_class_init (BuilderSourceClass *klass)
klass->show_deps = builder_source_real_show_deps;
klass->download = builder_source_real_download;
klass->extract = builder_source_real_extract;
klass->bundle = builder_source_real_bundle;
klass->update = builder_source_real_update;
g_object_class_install_property (object_class,
@ -322,6 +333,17 @@ builder_source_extract (BuilderSource *self,
return class->extract (self, real_dest, build_options, context, error);
}
gboolean
builder_source_bundle (BuilderSource *self,
BuilderContext *context,
GError **error)
{
BuilderSourceClass *class;
class = BUILDER_SOURCE_GET_CLASS (self);
return class->bundle (self, context, error);
}
gboolean
builder_source_update (BuilderSource *self,
BuilderContext *context,

View File

@ -61,6 +61,9 @@ typedef struct
BuilderOptions *build_options,
BuilderContext *context,
GError **error);
gboolean (* bundle)(BuilderSource *self,
BuilderContext *context,
GError **error);
gboolean (* update)(BuilderSource *self,
BuilderContext *context,
GError **error);
@ -85,6 +88,9 @@ gboolean builder_source_extract (BuilderSource *self,
BuilderOptions *build_options,
BuilderContext *context,
GError **error);
gboolean builder_source_bundle (BuilderSource *self,
BuilderContext *context,
GError **error);
gboolean builder_source_update (BuilderSource *self,
BuilderContext *context,
GError **error);