From 7cfdec0538abce06219c7cc1b550f8ff83c47d3c Mon Sep 17 00:00:00 2001
From: Martin Pilarski <38889214+cxrvh@users.noreply.github.com>
Date: Fri, 15 Jun 2018 11:14:39 +0200
Subject: [PATCH] Use GInputStream for checksum computation of source files &
 archives

Using g_load_contents () for the checksum computation uses a lot of memory
(for large files) and actually fails for files larger than 4 GiB. Instead,
this implementation uses GInputStream to read the file in small buffers
and feeds the GChecksum manually.

Closes: #165
Approved by: alexlarsson
---
 src/builder-source-archive.c | 19 ++++------------
 src/builder-source-file.c    | 19 ++++------------
 src/builder-utils.c          | 44 ++++++++++++++++++++++++++++++++----
 src/builder-utils.h          |  3 +--
 4 files changed, 48 insertions(+), 37 deletions(-)

diff --git a/src/builder-source-archive.c b/src/builder-source-archive.c
index b06e9a99..d35c67f5 100644
--- a/src/builder-source-archive.c
+++ b/src/builder-source-archive.c
@@ -397,21 +397,10 @@ builder_source_archive_download (BuilderSource  *source,
 
   if (g_file_query_exists (file, NULL))
     {
-      if (is_local && checksums[0]  != NULL)
-        {
-          g_autofree char *data = NULL;
-          gsize len;
-
-          if (!g_file_load_contents (file, NULL, &data, &len, NULL, error))
-            return FALSE;
-
-          if (!builder_verify_checksums (base_name,
-                                         data, len,
-                                         checksums, checksums_type,
-                                         error))
-            return FALSE;
-        }
-      return TRUE;
+      return !is_local || checksums[0] == NULL ||
+             builder_verify_checksums (base_name, file,
+                                       checksums, checksums_type,
+                                       error);
     }
 
   if (is_local)
diff --git a/src/builder-source-file.c b/src/builder-source-file.c
index 04f0a5f2..8a8d3897 100644
--- a/src/builder-source-file.c
+++ b/src/builder-source-file.c
@@ -361,21 +361,10 @@ builder_source_file_download (BuilderSource  *source,
 
   if (g_file_query_exists (file, NULL))
     {
-      if (is_local && checksums[0] != NULL)
-        {
-          g_autofree char *data = NULL;
-          gsize len;
-
-          if (!g_file_load_contents (file, NULL, &data, &len, NULL, error))
-            return FALSE;
-
-          if (!builder_verify_checksums (base_name,
-                                         data, len,
-                                         checksums, checksums_type,
-                                         error))
-            return FALSE;
-        }
-      return TRUE;
+      return !is_local || checksums[0] == NULL ||
+             builder_verify_checksums (base_name, file,
+                                       checksums, checksums_type,
+                                       error);
     }
 
   if (is_local)
diff --git a/src/builder-utils.c b/src/builder-utils.c
index 966987b1..aa09dd3f 100644
--- a/src/builder-utils.c
+++ b/src/builder-utils.c
@@ -1857,10 +1857,11 @@ compare_checksum (const char *name,
   return TRUE;
 }
 
+#define GET_BUFFER_SIZE 8192
+
 gboolean
 builder_verify_checksums (const char *name,
-                          const char *data,
-                          gsize len,
+                          GFile *file,
                           const char *checksums[BUILDER_CHECKSUMS_LEN],
                           GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
                           GError **error)
@@ -1869,9 +1870,42 @@ builder_verify_checksums (const char *name,
 
   for (i = 0; checksums[i] != NULL; i++)
     {
-      g_autofree char *checksum = NULL;
-      checksum = g_compute_checksum_for_string (checksums_type[i], data, len);
-      if (!compare_checksum (name, checksums[i], checksums_type[i], checksum, error))
+      g_autoptr(GFileInputStream) stream = NULL;
+
+      stream = g_file_read (file, NULL, error);
+
+      if (stream == NULL)
+        return FALSE;
+
+      gssize bytes_read;
+      guchar buffer[GET_BUFFER_SIZE];
+      GChecksum *checksum = NULL;
+      const char *checksum_string = NULL;
+      gboolean is_valid;
+
+      checksum = g_checksum_new (checksums_type[i]);
+
+      while ((bytes_read = g_input_stream_read (G_INPUT_STREAM(stream),
+                                                buffer, GET_BUFFER_SIZE,
+                                                NULL, error)) > 0)
+        {
+          g_checksum_update (checksum, buffer, bytes_read);
+        }
+
+      if (bytes_read < 0)
+        {
+          is_valid = FALSE;
+        }
+      else
+        {
+          checksum_string = g_checksum_get_string (checksum);
+          is_valid = compare_checksum (name, checksums[i], checksums_type[i],
+                                       checksum_string, error);
+        }
+
+      g_checksum_free (checksum);
+
+      if (!is_valid)
         return FALSE;
     }
 
diff --git a/src/builder-utils.h b/src/builder-utils.h
index 9412b853..266069f2 100644
--- a/src/builder-utils.h
+++ b/src/builder-utils.h
@@ -99,8 +99,7 @@ gsize builder_get_all_checksums (const char *checksums[BUILDER_CHECKSUMS_LEN],
                                  const char *sha512);
 
 gboolean builder_verify_checksums (const char *name,
-                                   const char *data,
-                                   gsize len,
+                                   GFile *file,
                                    const char *checksums[BUILDER_CHECKSUMS_LEN],
                                    GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
                                    GError **error);