marko10_000
/
flatpak-builder
forked from Mirrors/flatpak-builder
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for FTP sources using libcurl 

Closes: #143
Approved by: TingPing
auto
Denis Ollier 2018-05-03 09:28:27 +02:00 committed by Atomic Bot
parent 36f620240e
commit a922cd49a4
7 changed files with 173 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configure.ac
View File

@ -94,7 +94,7 @@ AC_CHECK_FUNCS(fdwalk)
AC_CHECK_HEADER([sys/xattr.h], [], [AC_MSG_ERROR([You must have sys/xattr.h from glibc])])
AC_CHECK_HEADER([sys/capability.h], have_caps=yes, [AC_MSG_ERROR([sys/capability.h header not found])])
PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4])
PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4 libcurl])
dnl ************************
dnl *** check for libelf ***

19
src/builder-context.c
View File

@ -46,6 +46,7 @@ struct BuilderContext
GFile *base_dir; /* directory with json manifest, origin for source files */
char *state_subdir;
SoupSession *soup_session;
CURL *curl_session;
char *arch;
char *stop_at;
@ -126,6 +127,10 @@ builder_context_finalize (GObject *object)
g_clear_pointer (&self->sources_dirs, g_ptr_array_unref);
g_clear_pointer (&self->sources_urls, g_ptr_array_unref);
curl_easy_cleanup (self->curl_session);
self->curl_session = NULL;
curl_global_cleanup ();
G_OBJECT_CLASS (builder_context_parent_class)->finalize (object);
}
@ -387,10 +392,12 @@ builder_context_download_uri (BuilderContext *self,
dest,
checksums, checksums_type,
builder_context_get_soup_session (self),
builder_context_get_curl_session (self),
&my_error))
return TRUE;
if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND))
if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND) &&
!g_error_matches (my_error, BUILDER_CURL_ERROR, CURLE_REMOTE_FILE_NOT_FOUND))
g_warning ("Error downloading from mirror: %s\n", my_error->message);
}
}
@ -399,6 +406,7 @@ builder_context_download_uri (BuilderContext *self,
dest,
checksums, checksums_type,
builder_context_get_soup_session (self),
builder_context_get_curl_session (self),
error))
return FALSE;
@ -498,6 +506,15 @@ builder_context_get_soup_session (BuilderContext *self)
return self->soup_session;
}
CURL *
builder_context_get_curl_session (BuilderContext *self)
{
if (self->curl_session == NULL)
self->curl_session = flatpak_create_curl_session ("flatpak-builder " PACKAGE_VERSION);
return self->curl_session;
}
const char *
builder_context_get_arch (BuilderContext *self)
{

2
src/builder-context.h
View File

@ -23,6 +23,7 @@
#include <gio/gio.h>
#include <libsoup/soup.h>
#include <curl/curl.h>
#include "builder-options.h"
#include "builder-utils.h"
#include "builder-sdk-config.h"
@ -68,6 +69,7 @@ gboolean builder_context_download_uri (BuilderContext *self,
GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
GError **error);
SoupSession * builder_context_get_soup_session (BuilderContext *self);
CURL * builder_context_get_curl_session (BuilderContext *self);
const char * builder_context_get_arch (BuilderContext *self);
void builder_context_set_arch (BuilderContext *self,
const char *arch);

18
src/builder-flatpak-utils.c
View File

@ -1230,6 +1230,24 @@ flatpak_create_soup_session (const char *user_agent)
return soup_session;
}
CURL *
flatpak_create_curl_session (const char *user_agent)
{
CURL *curl_session;
curl_global_init (CURL_GLOBAL_DEFAULT);
curl_session = curl_easy_init ();
if (curl_session == NULL)
return NULL;
curl_easy_setopt (curl_session, CURLOPT_CONNECTTIMEOUT, 60);
curl_easy_setopt (curl_session, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt (curl_session, CURLOPT_USERAGENT, user_agent);
return curl_session;
}
gboolean
flatpak_download_http_uri (SoupSession *soup_session,
const char *uri,

2
src/builder-flatpak-utils.h
View File

@ -29,6 +29,7 @@
#include <libsoup/soup.h>
#include <ostree.h>
#include <json-glib/json-glib.h>
#include <curl/curl.h>
typedef enum {
FLATPAK_HOST_COMMAND_FLAGS_CLEAR_ENV = 1 << 0,
@ -349,6 +350,7 @@ gboolean flatpak_allocate_tmpdir (int tmpdir_dfd,
SoupSession * flatpak_create_soup_session (const char *user_agent);
CURL * flatpak_create_curl_session (const char *user_agent);
GBytes * flatpak_load_http_uri (SoupSession *soup_session,
const char *uri,
const char *etag,

162
src/builder-utils.c
View File

@ -39,6 +39,7 @@
#include "builder-flatpak-utils.h"
#include "builder-utils.h"
G_DEFINE_QUARK (builder-curl-error, builder_curl_error)
G_DEFINE_QUARK (builder-yaml-parse-error, builder_yaml_parse_error)
#ifdef FLATPAK_BUILDER_ENABLE_YAML
@ -1877,6 +1878,62 @@ builder_verify_checksums (const char *name,
return TRUE;
}
typedef struct {
GOutputStream *out;
GChecksum **checksums;
gsize n_checksums;
GError **error;
} CURLWriteData;
static gsize
builder_curl_write_cb (gpointer *buffer,
gsize size,
gsize nmemb,
gpointer *userdata)
{
gsize bytes_written;
CURLWriteData *write_data = (CURLWriteData *) userdata;
flatpak_write_update_checksum (write_data->out, buffer, size * nmemb, &bytes_written,
write_data->checksums, write_data->n_checksums,
NULL, write_data->error);
return bytes_written;
}
static gboolean
builder_download_uri_curl (SoupURI *uri,
CURL *session,
GOutputStream *out,
GChecksum **checksums,
gsize n_checksums,
GError **error)
{
CURLcode retcode;
CURLWriteData write_data;
g_autofree gchar *url = soup_uri_to_string (uri, FALSE);
curl_easy_setopt (session, CURLOPT_URL, url);
curl_easy_setopt (session, CURLOPT_WRITEFUNCTION, builder_curl_write_cb);
curl_easy_setopt (session, CURLOPT_WRITEDATA, &write_data);
write_data.out = out;
write_data.checksums = checksums;
write_data.n_checksums = n_checksums;
write_data.error = error;
retcode = curl_easy_perform (session);
if (retcode != CURLE_OK)
{
g_set_error_literal (error, BUILDER_CURL_ERROR, retcode,
curl_easy_strerror (retcode));
return FALSE;
}
return TRUE;
}
typedef struct {
int stage;
gboolean printed_something;
@ -1903,40 +1960,17 @@ download_progress_cleanup (DownloadPromptData *progress_data)
g_print ("\b");
}
gboolean
builder_download_uri (SoupURI *uri,
GFile *dest,
const char *checksums[BUILDER_CHECKSUMS_LEN],
GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
SoupSession *session,
GError **error)
static gboolean
builder_download_uri_soup (SoupURI *uri,
SoupSession *session,
GOutputStream *out,
GChecksum **checksums,
gsize n_checksums,
GError **error)
{
g_autoptr(SoupRequest) req = NULL;
g_autoptr(GInputStream) input = NULL;
g_autoptr(GFileOutputStream) out = NULL;
g_autoptr(GFile) tmp = NULL;
g_autoptr(GFile) dir = NULL;
g_autoptr(GPtrArray) checksum_array = g_ptr_array_new_with_free_func ((GDestroyNotify)g_checksum_free);
g_autoptr(SoupRequest) req = NULL;
DownloadPromptData progress_data = {0};
g_autofree char *basename = g_file_get_basename (dest);
g_autofree char *template = g_strconcat (".", basename, "XXXXXX", NULL);
gsize i;
for (i = 0; checksums[i] != NULL; i++)
g_ptr_array_add (checksum_array,
g_checksum_new (checksums_type[i]));
dir = g_file_get_parent (dest);
g_mkdir_with_parents (flatpak_file_get_path_cached (dir), 0755);
tmp = flatpak_file_new_tmp_in (dir, template, error);
if (tmp == NULL)
return FALSE;
out = g_file_replace (tmp, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION,
NULL, error);
if (out == NULL)
return FALSE;
req = soup_session_request_uri (session, uri, error);
if (req == NULL)
@ -1957,8 +1991,68 @@ builder_download_uri (SoupURI *uri,
}
if (!flatpak_splice_update_checksum (G_OUTPUT_STREAM (out), input,
(GChecksum **)checksum_array->pdata, checksum_array->len,
download_progress, &progress_data, NULL, error))
checksums, n_checksums,
download_progress, &progress_data,
NULL, error))
return FALSE;
download_progress_cleanup (&progress_data);
return TRUE;
}
gboolean
builder_download_uri (SoupURI *uri,
GFile *dest,
const char *checksums[BUILDER_CHECKSUMS_LEN],
GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
SoupSession *soup_session,
CURL *curl_session,
GError **error)
{
g_autoptr(GFileOutputStream) out = NULL;
g_autoptr(GFile) tmp = NULL;
g_autoptr(GFile) dir = NULL;
g_autoptr(GPtrArray) checksum_array = g_ptr_array_new_with_free_func ((GDestroyNotify)g_checksum_free);
g_autofree char *basename = g_file_get_basename (dest);
g_autofree char *template = g_strconcat (".", basename, "XXXXXX", NULL);
gsize i;
gboolean download_res;
for (i = 0; checksums[i] != NULL; i++)
g_ptr_array_add (checksum_array,
g_checksum_new (checksums_type[i]));
dir = g_file_get_parent (dest);
g_mkdir_with_parents (flatpak_file_get_path_cached (dir), 0755);
tmp = flatpak_file_new_tmp_in (dir, template, error);
if (tmp == NULL)
return FALSE;
out = g_file_replace (tmp, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION,
NULL, error);
if (out == NULL)
return FALSE;
if (SOUP_URI_VALID_FOR_HTTP(uri))
{
download_res = builder_download_uri_soup (uri, soup_session,
G_OUTPUT_STREAM (out),
(GChecksum **)checksum_array->pdata,
checksum_array->len,
error);
}
else
{
download_res = builder_download_uri_curl (uri, curl_session,
G_OUTPUT_STREAM (out),
(GChecksum **)checksum_array->pdata,
checksum_array->len,
error);
}
if (download_res == FALSE)
{
unlink (flatpak_file_get_path_cached (tmp));
return FALSE;
@ -1971,8 +2065,6 @@ builder_download_uri (SoupURI *uri,
return FALSE;
}
download_progress_cleanup (&progress_data);
for (i = 0; checksums[i] != NULL; i++)
{
const char *checksum = g_checksum_get_string (g_ptr_array_index (checksum_array, i));

5
src/builder-utils.h
View File

@ -24,6 +24,7 @@
#include <gio/gio.h>
#include <libsoup/soup.h>
#include <json-glib/json-glib.h>
#include <curl/curl.h>
#include <libxml/tree.h>
@ -61,6 +62,9 @@ void flatpak_collect_matches_for_path_pattern (const char *path,
gboolean builder_migrate_locale_dirs (GFile *root_dir,
GError **error);
GQuark builder_curl_error_quark (void);
#define BUILDER_CURL_ERROR (builder_curl_error_quark ())
GQuark builder_yaml_parse_error_quark (void);
#define BUILDER_YAML_PARSE_ERROR (builder_yaml_parse_error_quark ())
@ -85,6 +89,7 @@ gboolean builder_download_uri (SoupURI *uri,
const char *checksums[BUILDER_CHECKSUMS_LEN],
GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
SoupSession *soup_session,
CURL *curl_session,
GError **error);
gsize builder_get_all_checksums (const char *checksums[BUILDER_CHECKSUMS_LEN],