From a922cd49a46cca3f9d93e4cc90d6c2ee4858b485 Mon Sep 17 00:00:00 2001 From: Denis Ollier Date: Thu, 3 May 2018 09:28:27 +0200 Subject: [PATCH] Add support for FTP sources using libcurl Closes: #143 Approved by: TingPing --- configure.ac | 2 +- src/builder-context.c | 19 ++++- src/builder-context.h | 2 + src/builder-flatpak-utils.c | 18 ++++ src/builder-flatpak-utils.h | 2 + src/builder-utils.c | 162 ++++++++++++++++++++++++++++-------- src/builder-utils.h | 5 ++ 7 files changed, 173 insertions(+), 37 deletions(-) diff --git a/configure.ac b/configure.ac index cbaf0473..ca789265 100644 --- a/configure.ac +++ b/configure.ac @@ -94,7 +94,7 @@ AC_CHECK_FUNCS(fdwalk) AC_CHECK_HEADER([sys/xattr.h], [], [AC_MSG_ERROR([You must have sys/xattr.h from glibc])]) AC_CHECK_HEADER([sys/capability.h], have_caps=yes, [AC_MSG_ERROR([sys/capability.h header not found])]) -PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4]) +PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4 libcurl]) dnl ************************ dnl *** check for libelf *** diff --git a/src/builder-context.c b/src/builder-context.c index 19e9612f..cf4bdb41 100644 --- a/src/builder-context.c +++ b/src/builder-context.c @@ -46,6 +46,7 @@ struct BuilderContext GFile *base_dir; /* directory with json manifest, origin for source files */ char *state_subdir; SoupSession *soup_session; + CURL *curl_session; char *arch; char *stop_at; @@ -126,6 +127,10 @@ builder_context_finalize (GObject *object) g_clear_pointer (&self->sources_dirs, g_ptr_array_unref); g_clear_pointer (&self->sources_urls, g_ptr_array_unref); + curl_easy_cleanup (self->curl_session); + self->curl_session = NULL; + curl_global_cleanup (); + G_OBJECT_CLASS (builder_context_parent_class)->finalize (object); } @@ -387,10 +392,12 @@ builder_context_download_uri (BuilderContext *self, dest, checksums, checksums_type, builder_context_get_soup_session (self), + builder_context_get_curl_session (self), &my_error)) return TRUE; - if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND)) + if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND) && + !g_error_matches (my_error, BUILDER_CURL_ERROR, CURLE_REMOTE_FILE_NOT_FOUND)) g_warning ("Error downloading from mirror: %s\n", my_error->message); } } @@ -399,6 +406,7 @@ builder_context_download_uri (BuilderContext *self, dest, checksums, checksums_type, builder_context_get_soup_session (self), + builder_context_get_curl_session (self), error)) return FALSE; @@ -498,6 +506,15 @@ builder_context_get_soup_session (BuilderContext *self) return self->soup_session; } +CURL * +builder_context_get_curl_session (BuilderContext *self) +{ + if (self->curl_session == NULL) + self->curl_session = flatpak_create_curl_session ("flatpak-builder " PACKAGE_VERSION); + + return self->curl_session; +} + const char * builder_context_get_arch (BuilderContext *self) { diff --git a/src/builder-context.h b/src/builder-context.h index 2c1a7691..85ecd17b 100644 --- a/src/builder-context.h +++ b/src/builder-context.h @@ -23,6 +23,7 @@ #include #include +#include #include "builder-options.h" #include "builder-utils.h" #include "builder-sdk-config.h" @@ -68,6 +69,7 @@ gboolean builder_context_download_uri (BuilderContext *self, GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN], GError **error); SoupSession * builder_context_get_soup_session (BuilderContext *self); +CURL * builder_context_get_curl_session (BuilderContext *self); const char * builder_context_get_arch (BuilderContext *self); void builder_context_set_arch (BuilderContext *self, const char *arch); diff --git a/src/builder-flatpak-utils.c b/src/builder-flatpak-utils.c index f231e18f..7a3e3e76 100644 --- a/src/builder-flatpak-utils.c +++ b/src/builder-flatpak-utils.c @@ -1230,6 +1230,24 @@ flatpak_create_soup_session (const char *user_agent) return soup_session; } +CURL * +flatpak_create_curl_session (const char *user_agent) +{ + CURL *curl_session; + + curl_global_init (CURL_GLOBAL_DEFAULT); + + curl_session = curl_easy_init (); + if (curl_session == NULL) + return NULL; + + curl_easy_setopt (curl_session, CURLOPT_CONNECTTIMEOUT, 60); + curl_easy_setopt (curl_session, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt (curl_session, CURLOPT_USERAGENT, user_agent); + + return curl_session; +} + gboolean flatpak_download_http_uri (SoupSession *soup_session, const char *uri, diff --git a/src/builder-flatpak-utils.h b/src/builder-flatpak-utils.h index 116065d9..cef23318 100644 --- a/src/builder-flatpak-utils.h +++ b/src/builder-flatpak-utils.h @@ -29,6 +29,7 @@ #include #include #include +#include typedef enum { FLATPAK_HOST_COMMAND_FLAGS_CLEAR_ENV = 1 << 0, @@ -349,6 +350,7 @@ gboolean flatpak_allocate_tmpdir (int tmpdir_dfd, SoupSession * flatpak_create_soup_session (const char *user_agent); +CURL * flatpak_create_curl_session (const char *user_agent); GBytes * flatpak_load_http_uri (SoupSession *soup_session, const char *uri, const char *etag, diff --git a/src/builder-utils.c b/src/builder-utils.c index 3bcf4a3c..4f4b8da8 100644 --- a/src/builder-utils.c +++ b/src/builder-utils.c @@ -39,6 +39,7 @@ #include "builder-flatpak-utils.h" #include "builder-utils.h" +G_DEFINE_QUARK (builder-curl-error, builder_curl_error) G_DEFINE_QUARK (builder-yaml-parse-error, builder_yaml_parse_error) #ifdef FLATPAK_BUILDER_ENABLE_YAML @@ -1877,6 +1878,62 @@ builder_verify_checksums (const char *name, return TRUE; } +typedef struct { + GOutputStream *out; + GChecksum **checksums; + gsize n_checksums; + GError **error; +} CURLWriteData; + +static gsize +builder_curl_write_cb (gpointer *buffer, + gsize size, + gsize nmemb, + gpointer *userdata) +{ + gsize bytes_written; + CURLWriteData *write_data = (CURLWriteData *) userdata; + + flatpak_write_update_checksum (write_data->out, buffer, size * nmemb, &bytes_written, + write_data->checksums, write_data->n_checksums, + NULL, write_data->error); + + return bytes_written; +} + +static gboolean +builder_download_uri_curl (SoupURI *uri, + CURL *session, + GOutputStream *out, + GChecksum **checksums, + gsize n_checksums, + GError **error) +{ + CURLcode retcode; + CURLWriteData write_data; + g_autofree gchar *url = soup_uri_to_string (uri, FALSE); + + curl_easy_setopt (session, CURLOPT_URL, url); + curl_easy_setopt (session, CURLOPT_WRITEFUNCTION, builder_curl_write_cb); + curl_easy_setopt (session, CURLOPT_WRITEDATA, &write_data); + + write_data.out = out; + write_data.checksums = checksums; + write_data.n_checksums = n_checksums; + write_data.error = error; + + retcode = curl_easy_perform (session); + + if (retcode != CURLE_OK) + { + g_set_error_literal (error, BUILDER_CURL_ERROR, retcode, + curl_easy_strerror (retcode)); + return FALSE; + } + + return TRUE; +} + typedef struct { int stage; gboolean printed_something; @@ -1903,40 +1960,17 @@ download_progress_cleanup (DownloadPromptData *progress_data) g_print ("\b"); } -gboolean -builder_download_uri (SoupURI *uri, - GFile *dest, - const char *checksums[BUILDER_CHECKSUMS_LEN], - GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN], - SoupSession *session, - GError **error) +static gboolean +builder_download_uri_soup (SoupURI *uri, + SoupSession *session, + GOutputStream *out, + GChecksum **checksums, + gsize n_checksums, + GError **error) { - g_autoptr(SoupRequest) req = NULL; g_autoptr(GInputStream) input = NULL; - g_autoptr(GFileOutputStream) out = NULL; - g_autoptr(GFile) tmp = NULL; - g_autoptr(GFile) dir = NULL; - g_autoptr(GPtrArray) checksum_array = g_ptr_array_new_with_free_func ((GDestroyNotify)g_checksum_free); + g_autoptr(SoupRequest) req = NULL; DownloadPromptData progress_data = {0}; - g_autofree char *basename = g_file_get_basename (dest); - g_autofree char *template = g_strconcat (".", basename, "XXXXXX", NULL); - gsize i; - - for (i = 0; checksums[i] != NULL; i++) - g_ptr_array_add (checksum_array, - g_checksum_new (checksums_type[i])); - - dir = g_file_get_parent (dest); - g_mkdir_with_parents (flatpak_file_get_path_cached (dir), 0755); - - tmp = flatpak_file_new_tmp_in (dir, template, error); - if (tmp == NULL) - return FALSE; - - out = g_file_replace (tmp, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION, - NULL, error); - if (out == NULL) - return FALSE; req = soup_session_request_uri (session, uri, error); if (req == NULL) @@ -1957,8 +1991,68 @@ builder_download_uri (SoupURI *uri, } if (!flatpak_splice_update_checksum (G_OUTPUT_STREAM (out), input, - (GChecksum **)checksum_array->pdata, checksum_array->len, - download_progress, &progress_data, NULL, error)) + checksums, n_checksums, + download_progress, &progress_data, + NULL, error)) + return FALSE; + + download_progress_cleanup (&progress_data); + + return TRUE; +} + +gboolean +builder_download_uri (SoupURI *uri, + GFile *dest, + const char *checksums[BUILDER_CHECKSUMS_LEN], + GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN], + SoupSession *soup_session, + CURL *curl_session, + GError **error) +{ + g_autoptr(GFileOutputStream) out = NULL; + g_autoptr(GFile) tmp = NULL; + g_autoptr(GFile) dir = NULL; + g_autoptr(GPtrArray) checksum_array = g_ptr_array_new_with_free_func ((GDestroyNotify)g_checksum_free); + g_autofree char *basename = g_file_get_basename (dest); + g_autofree char *template = g_strconcat (".", basename, "XXXXXX", NULL); + gsize i; + gboolean download_res; + + for (i = 0; checksums[i] != NULL; i++) + g_ptr_array_add (checksum_array, + g_checksum_new (checksums_type[i])); + + dir = g_file_get_parent (dest); + g_mkdir_with_parents (flatpak_file_get_path_cached (dir), 0755); + + tmp = flatpak_file_new_tmp_in (dir, template, error); + if (tmp == NULL) + return FALSE; + + out = g_file_replace (tmp, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION, + NULL, error); + if (out == NULL) + return FALSE; + + if (SOUP_URI_VALID_FOR_HTTP(uri)) + { + download_res = builder_download_uri_soup (uri, soup_session, + G_OUTPUT_STREAM (out), + (GChecksum **)checksum_array->pdata, + checksum_array->len, + error); + } + else + { + download_res = builder_download_uri_curl (uri, curl_session, + G_OUTPUT_STREAM (out), + (GChecksum **)checksum_array->pdata, + checksum_array->len, + error); + } + + if (download_res == FALSE) { unlink (flatpak_file_get_path_cached (tmp)); return FALSE; @@ -1971,8 +2065,6 @@ builder_download_uri (SoupURI *uri, return FALSE; } - download_progress_cleanup (&progress_data); - for (i = 0; checksums[i] != NULL; i++) { const char *checksum = g_checksum_get_string (g_ptr_array_index (checksum_array, i)); diff --git a/src/builder-utils.h b/src/builder-utils.h index d1b8ad38..f5b46496 100644 --- a/src/builder-utils.h +++ b/src/builder-utils.h @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -61,6 +62,9 @@ void flatpak_collect_matches_for_path_pattern (const char *path, gboolean builder_migrate_locale_dirs (GFile *root_dir, GError **error); +GQuark builder_curl_error_quark (void); +#define BUILDER_CURL_ERROR (builder_curl_error_quark ()) + GQuark builder_yaml_parse_error_quark (void); #define BUILDER_YAML_PARSE_ERROR (builder_yaml_parse_error_quark ()) @@ -85,6 +89,7 @@ gboolean builder_download_uri (SoupURI *uri, const char *checksums[BUILDER_CHECKSUMS_LEN], GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN], SoupSession *soup_session, + CURL *curl_session, GError **error); gsize builder_get_all_checksums (const char *checksums[BUILDER_CHECKSUMS_LEN],