marko10_000
/
flatpak-builder
forked from Mirrors/flatpak-builder
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

builder: Download files and archives directly to disk, don't keep them in memory. 

This works much better for larger sources.
tingping/wmclass
Alexander Larsson 2017-02-09 12:08:12 +01:00
parent bf316c1a82
commit 08ea3baf49
6 changed files with 107 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
builder/builder-source-archive.c
View File

@ -265,39 +265,6 @@ get_source_file (BuilderSourceArchive *self,
return NULL;
}
static GBytes *
download_uri (const char *url,
GChecksum *checksum,
BuilderContext *context,
GError **error)
{
SoupSession *session;
g_autoptr(SoupRequest) req = NULL;
g_autoptr(GInputStream) input = NULL;
g_autoptr(GOutputStream) out = NULL;
session = builder_context_get_soup_session (context);
req = soup_session_request (session, url, error);
if (req == NULL)
return NULL;
input = soup_request_send (req, NULL, error);
if (input == NULL)
return NULL;
out = g_memory_output_stream_new_resizable ();
if (!flatpak_splice_update_checksum (out, input, checksum, NULL, error))
return NULL;
/* Manually close to flush and detect write errors */
if (!g_output_stream_close (out, NULL, error))
return NULL;
return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (out));
}
static gboolean
builder_source_archive_show_deps (BuilderSource *source,
GError **error)
@ -310,7 +277,6 @@ builder_source_archive_show_deps (BuilderSource *source,
return TRUE;
}
static gboolean
builder_source_archive_download (BuilderSource *source,
gboolean update_vcs,
@ -325,7 +291,6 @@ builder_source_archive_download (BuilderSource *source,
const char *sha256 = NULL;
g_autofree char *base_name = NULL;
g_autoptr(GBytes) content = NULL;
g_autoptr(GChecksum) checksum = NULL;
gboolean is_local;
file = get_source_file (self, context, &is_local, error);
@ -361,35 +326,12 @@ builder_source_archive_download (BuilderSource *source,
return FALSE;
}
checksum = g_checksum_new (G_CHECKSUM_SHA256);
g_print ("Downloading %s\n", self->url);
content = download_uri (self->url,
checksum,
context,
error);
if (content == NULL)
return FALSE;
base_name = g_file_get_basename (file);
sha256 = g_checksum_get_string (checksum);
if (strcmp (sha256, self->sha256) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Wrong sha256 for %s, expected %s, was %s", base_name, self->sha256, sha256);
return FALSE;
}
dir = g_file_get_parent (file);
dir_path = g_file_get_path (dir);
g_mkdir_with_parents (dir_path, 0755);
if (!g_file_replace_contents (file,
g_bytes_get_data (content, NULL),
g_bytes_get_size (content),
NULL, FALSE, G_FILE_CREATE_NONE, NULL,
NULL, error))
if (!builder_download_uri (self->url,
file,
self->sha256,
builder_context_get_soup_session (context),
error))
return FALSE;
return TRUE;

31
builder/builder-source-file.c
View File

@ -318,34 +318,17 @@ builder_source_file_download (BuilderSource *source,
return FALSE;
}
content = download_uri (self->url,
context,
error);
if (content == NULL)
return FALSE;
sha256 = g_compute_checksum_for_string (G_CHECKSUM_SHA256,
g_bytes_get_data (content, NULL),
g_bytes_get_size (content));
/* sha256 is optional for inline data */
if (((self->sha256 != NULL && *self->sha256 != 0) || !is_inline) &&
strcmp (sha256, self->sha256) != 0)
if ((self->sha256 == NULL || *self->sha256 == 0) && !is_inline)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Wrong sha256 for %s, expected %s, was %s", base_name, self->sha256, sha256);
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Sha256 not specified");
return FALSE;
}
dir = g_file_get_parent (file);
dir_path = g_file_get_path (dir);
g_mkdir_with_parents (dir_path, 0755);
if (!g_file_replace_contents (file,
g_bytes_get_data (content, NULL),
g_bytes_get_size (content),
NULL, FALSE, G_FILE_CREATE_NONE, NULL,
NULL, error))
if (!builder_download_uri (self->url,
file,
self->sha256,
builder_context_get_soup_session (context),
error))
return FALSE;
return TRUE;

68
builder/builder-utils.c
View File

@ -26,6 +26,7 @@
#include <gelf.h>
#include <dwarf.h>
#include <sys/mman.h>
#include <stdio.h>
#include <string.h>
@ -1598,3 +1599,70 @@ builder_maybe_host_spawnv (GFile *dir,
return flatpak_spawnv (dir, output, error, argv);
}
gboolean
builder_download_uri (const char *url,
GFile *dest,
char *sha256,
SoupSession *session,
GError **error)
{
g_autoptr(SoupRequest) req = NULL;
g_autoptr(GInputStream) input = NULL;
g_autoptr(GFileOutputStream) out = NULL;
g_autoptr(GFile) tmp = NULL;
g_autoptr(GFile) dir = NULL;
g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256);
g_autofree char *basename = g_file_get_basename (dest);
g_autofree char *template = g_strconcat (".", basename, "XXXXXX", NULL);
dir = g_file_get_parent (dest);
g_mkdir_with_parents (flatpak_file_get_path_cached (dir), 0755);
tmp = flatpak_file_new_tmp_in (dir, template, error);
if (tmp == NULL)
return FALSE;
out = g_file_replace (tmp, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION,
NULL, error);
if (out == NULL)
return FALSE;
req = soup_session_request (session, url, error);
if (req == NULL)
return FALSE;
input = soup_request_send (req, NULL, error);
if (input == NULL)
return FALSE;
if (!flatpak_splice_update_checksum (G_OUTPUT_STREAM (out), input, checksum, NULL, error))
{
unlink (flatpak_file_get_path_cached (tmp));
return FALSE;
}
/* Manually close to flush and detect write errors */
if (!g_output_stream_close (G_OUTPUT_STREAM (out), NULL, error))
{
unlink (flatpak_file_get_path_cached (tmp));
return FALSE;
}
if (sha256 != NULL && strcmp (g_checksum_get_string (checksum), sha256) != 0)
{
unlink (flatpak_file_get_path_cached (tmp));
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Wrong sha256 for %s, expected %s, was %s", basename, sha256, g_checksum_get_string (checksum));
return FALSE;
}
if (rename (flatpak_file_get_path_cached (tmp), flatpak_file_get_path_cached (dest)) != 0)
{
glnx_set_error_from_errno (error);
return FALSE;
}
return TRUE;
}

5
builder/builder-utils.h
View File

@ -62,6 +62,11 @@ gboolean builder_maybe_host_spawnv (GFile *dir,
GError **error,
const gchar * const *argv);
gboolean builder_download_uri (const char *url,
GFile *dest,
char *sha256,
SoupSession *soup_session,
GError **error);
G_END_DECLS

18
common/flatpak-utils.c
View File

@ -74,6 +74,24 @@ flatpak_error_quark (void)
return (GQuark) quark_volatile;
}
GFile *
flatpak_file_new_tmp_in (GFile *dir,
const char *template,
GError **error)
{
glnx_fd_close int tmp_fd = -1;
g_autofree char *tmpl = g_build_filename (flatpak_file_get_path_cached (dir), template, NULL);
tmp_fd = g_mkstemp_full (tmpl, O_RDWR, 0644);
if (tmp_fd == -1)
{
glnx_set_error_from_errno (error);
return NULL;
}
return g_file_new_for_path (tmpl);
}
gboolean
flatpak_write_update_checksum (GOutputStream *out,
gconstpointer data,

4
common/flatpak-utils.h
View File

@ -75,6 +75,10 @@ char ** flatpak_get_current_locale_subpaths (void);
void flatpak_migrate_from_xdg_app (void);
GFile *flatpak_file_new_tmp_in (GFile *dir,
const char *templatename,
GError **error);
gboolean flatpak_write_update_checksum (GOutputStream *out,
gconstpointer data,
gsize len,