Add support for running tests

Modules that say "run-tests": true, will run tests after installation,
unless disabled by --disable-tests.

The tests run by default are make check or ninja test, however you
can control the make/ninja target with test-rule, or supply a list
of commands with test-commands. There is also a test-args argument
in build-options, which you can use to give e.g. network access.

The tests are run with readonly access to the install directory, so
they cannot affect the build results.

Closes: #65
Approved by: alexlarsson

doc/flatpak-builder.xml

@@ -203,6 +203,14 @@
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--disable-tests</option></term>
+
+                <listitem><para>
+                    Don't run any of the tests.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--run</option></term>
 

doc/flatpak-manifest.xml

@@ -289,6 +289,10 @@
                     <term><option>build-args</option> (array of strings)</term>
                     <listitem><para>This is an array containing extra options to pass to flatpak build.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>test-args</option> (array of strings)</term>
+                    <listitem><para>Similar to build-args but affects the tests, not the normal build.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>config-opts</option> (array of strings)</term>
                     <listitem><para>This is an array containing extra options to pass to configure.</para></listitem>
@@ -470,6 +474,18 @@
                     <term><option>cleanup-platform</option> (array of strings)</term>
                     <listitem><para>Extra files to clean up in the platform.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>run-tests</option> (boolean)</term>
+                    <listitem><para>If true this will run the tests after installing.</para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>test-rule</option> (string)</term>
+                    <listitem><para>The target to build when running the tests. Defaults to "check" for make and "test" for ninja. Set to empty to disable.</para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>test-commands</option> (arrya of string)</term>
+                    <listitem><para>Array of commands to run during the tests.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>modules</option> (array of objects or strings)</term>
                     <listitem><para>An array of objects specifying nested modules to be built before this one.

src/builder-context.c

@@ -75,6 +75,7 @@ struct BuilderContext
   gboolean        rebuild_on_sdk_change;
   gboolean        use_rofiles;
   gboolean        have_rofiles;
+  gboolean        run_tests;
 };
 
 typedef struct
@@ -826,6 +827,19 @@ builder_context_set_use_rofiles (BuilderContext *self,
   self->use_rofiles = use_rofiles;
 }
 
+gboolean
+builder_context_get_run_tests (BuilderContext *self)
+{
+  return self->run_tests;
+}
+
+void
+builder_context_set_run_tests (BuilderContext *self,
+                               gboolean run_tests)
+{
+  self->run_tests = run_tests;
+}
+
 gboolean
 builder_context_get_rebuild_on_sdk_change (BuilderContext *self)
 {

src/builder-context.h

@@ -125,6 +125,9 @@ gboolean        builder_context_get_rofiles_active (BuilderContext *self);
 gboolean        builder_context_get_use_rofiles (BuilderContext *self);
 void            builder_context_set_use_rofiles (BuilderContext *self,
                                                  gboolean use_rofiles);
+gboolean        builder_context_get_run_tests (BuilderContext *self);
+void            builder_context_set_run_tests (BuilderContext *self,
+                                               gboolean run_tests);
 char **         builder_context_extend_env (BuilderContext *self,
                                             char          **envp);
 

src/builder-main.c

@@ -38,6 +38,7 @@ static gboolean opt_verbose;
 static gboolean opt_version;
 static gboolean opt_run;
 static gboolean opt_disable_cache;
+static gboolean opt_disable_tests;
 static gboolean opt_disable_rofiles;
 static gboolean opt_download_only;
 static gboolean opt_bundle_sources;
@@ -87,6 +88,7 @@ static GOptionEntry entries[] = {
   { "run", 0, 0, G_OPTION_ARG_NONE, &opt_run, "Run a command in the build directory (see --run --help)", NULL },
   { "ccache", 0, 0, G_OPTION_ARG_NONE, &opt_ccache, "Use ccache", NULL },
   { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Disable cache lookups", NULL },
+  { "disable-tests", 0, 0, G_OPTION_ARG_NONE, &opt_disable_tests, "Don't run tests", NULL },
   { "disable-rofiles-fuse", 0, 0, G_OPTION_ARG_NONE, &opt_disable_rofiles, "Disable rofiles-fuse use", NULL },
   { "disable-download", 0, 0, G_OPTION_ARG_NONE, &opt_disable_download, "Don't download any new sources", NULL },
   { "disable-updates", 0, 0, G_OPTION_ARG_NONE, &opt_disable_updates, "Only download missing sources, never update to latest vcs version", NULL },
@@ -377,6 +379,7 @@ main (int    argc,
   build_context = builder_context_new (cwd_dir, app_dir);
 
   builder_context_set_use_rofiles (build_context, !opt_disable_rofiles);
+  builder_context_set_run_tests (build_context, !opt_disable_tests);
   builder_context_set_keep_build_dirs (build_context, opt_keep_build_dirs);
   builder_context_set_delete_build_dirs (build_context, opt_delete_build_dirs);
   builder_context_set_sandboxed (build_context, opt_sandboxed);

src/builder-module.c

@@ -49,6 +49,7 @@ struct BuilderModule
   char          **make_args;
   char          **make_install_args;
   char           *install_rule;
+  char           *test_rule;
   char           *buildsystem;
   char          **ensure_writable;
   char          **only_arches;
@@ -61,6 +62,7 @@ struct BuilderModule
   gboolean        no_python_timestamp_fix;
   gboolean        cmake;
   gboolean        builddir;
+  gboolean        run_tests;
   BuilderOptions *build_options;
   GPtrArray      *changes;
   char          **cleanup;
@@ -68,6 +70,7 @@ struct BuilderModule
   GList          *sources;
   GList          *modules;
   char          **build_commands;
+  char          **test_commands;
 };
 
 typedef struct
@@ -92,6 +95,7 @@ enum {
   PROP_NO_PYTHON_TIMESTAMP_FIX,
   PROP_CMAKE,
   PROP_INSTALL_RULE,
+  PROP_TEST_RULE,
   PROP_BUILDSYSTEM,
   PROP_BUILDDIR,
   PROP_CONFIG_OPTS,
@@ -99,6 +103,7 @@ enum {
   PROP_MAKE_INSTALL_ARGS,
   PROP_ENSURE_WRITABLE,
   PROP_ONLY_ARCHES,
+  PROP_RUN_TESTS,
   PROP_SKIP_ARCHES,
   PROP_SOURCES,
   PROP_BUILD_OPTIONS,
@@ -107,6 +112,7 @@ enum {
   PROP_POST_INSTALL,
   PROP_MODULES,
   PROP_BUILD_COMMANDS,
+  PROP_TEST_COMMANDS,
   LAST_PROP
 };
 
@@ -134,6 +140,7 @@ builder_module_finalize (GObject *object)
   g_free (self->name);
   g_free (self->subdir);
   g_free (self->install_rule);
+  g_free (self->test_rule);
   g_free (self->buildsystem);
   g_strfreev (self->post_install);
   g_strfreev (self->config_opts);
@@ -148,6 +155,7 @@ builder_module_finalize (GObject *object)
   g_strfreev (self->cleanup_platform);
   g_list_free_full (self->modules, g_object_unref);
   g_strfreev (self->build_commands);
+  g_strfreev (self->test_commands);
 
   if (self->changes)
     g_ptr_array_unref (self->changes);
@@ -209,6 +217,10 @@ builder_module_get_property (GObject    *object,
       g_value_set_string (value, self->install_rule);
       break;
 
+    case PROP_TEST_RULE:
+      g_value_set_string (value, self->test_rule);
+      break;
+
     case PROP_BUILDDIR:
       g_value_set_boolean (value, self->builddir);
       break;
@@ -265,6 +277,14 @@ builder_module_get_property (GObject    *object,
       g_value_set_boxed (value, self->build_commands);
       break;
 
+    case PROP_TEST_COMMANDS:
+      g_value_set_boxed (value, self->test_commands);
+      break;
+
+    case PROP_RUN_TESTS:
+      g_value_set_boolean (value, self->run_tests);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -333,6 +353,11 @@ builder_module_set_property (GObject      *object,
       self->install_rule = g_value_dup_string (value);
       break;
 
+    case PROP_TEST_RULE:
+      g_free (self->test_rule);
+      self->test_rule = g_value_dup_string (value);
+      break;
+
     case PROP_BUILDDIR:
       self->builddir = g_value_get_boolean (value);
       break;
@@ -413,6 +438,16 @@ builder_module_set_property (GObject      *object,
       g_strfreev (tmp);
       break;
 
+    case PROP_TEST_COMMANDS:
+      tmp = self->test_commands;
+      self->test_commands = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
+    case PROP_RUN_TESTS:
+      self->run_tests = g_value_get_boolean (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -504,6 +539,13 @@ builder_module_class_init (BuilderModuleClass *klass)
                                                          "",
                                                          NULL,
                                                          G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_TEST_RULE,
+                                   g_param_spec_string ("test-rule",
+                                                         "",
+                                                         "",
+                                                         NULL,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_BUILDDIR,
                                    g_param_spec_boolean ("builddir",
@@ -600,6 +642,20 @@ builder_module_class_init (BuilderModuleClass *klass)
                                                        "",
                                                        G_TYPE_STRV,
                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_TEST_COMMANDS,
+                                   g_param_spec_boxed ("test-commands",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_RUN_TESTS,
+                                   g_param_spec_boolean ("run-tests",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
 }
 
 static void
@@ -1256,6 +1312,7 @@ builder_module_build_helper (BuilderModule  *self,
   g_autofree char *make_j = NULL;
   g_autofree char *make_l = NULL;
   const char *make_cmd = NULL;
+  const char *test_arg = NULL;
 
   gboolean autotools = FALSE, cmake = FALSE, cmake_ninja = FALSE, meson = FALSE, simple = FALSE, qmake = FALSE;
   g_autoptr(GFile) configure_file = NULL;
@@ -1583,11 +1640,20 @@ builder_module_build_helper (BuilderModule  *self,
   builder_set_term_title (_("Installing %s"), self->name);
 
   if (meson || cmake_ninja)
-    make_cmd = "ninja";
+    {
+      make_cmd = "ninja";
+      test_arg = "test";
+    }
   else if (simple)
     make_cmd = NULL;
   else
-    make_cmd = "make";
+    {
+      make_cmd = "make";
+      test_arg = "check";
+    }
+
+  if (self->test_rule)
+    test_arg = self->test_rule;
 
   if (make_cmd)
     {
@@ -1645,6 +1711,41 @@ builder_module_build_helper (BuilderModule  *self,
         }
     }
 
+  /* Run unit tests */
+
+  if (self->run_tests && builder_context_get_run_tests (context))
+    {
+      g_auto(GStrv) test_args = NULL;
+
+      builder_set_term_title (_("Testing %s"), self->name);
+      g_print ("Running tests\n");
+
+      test_args = builder_options_get_test_args (self->build_options, context, error);
+      if (test_args == NULL)
+        return FALSE;
+
+      if (make_cmd && test_arg && *test_arg != 0)
+        {
+          if (!build (app_dir, self->name, context, source_dir, build_dir_relative, test_args, env, error,
+                      make_cmd, test_arg, NULL))
+            {
+              g_prefix_error (error, "Running %s %s failed: ", make_cmd, test_arg);
+              return FALSE;
+            }
+        }
+
+      for (i = 0; self->test_commands != NULL && self->test_commands[i] != NULL; i++)
+        {
+          g_print ("Running: %s\n", self->test_commands[i]);
+          if (!build (app_dir, self->name, context, source_dir, build_dir_relative, test_args, env, error,
+                      "/bin/sh", "-c", self->test_commands[i], NULL))
+            {
+              g_prefix_error (error, "Running test command '%s' failed: ", self->test_commands[i]);
+              return FALSE;
+            }
+        }
+    }
+
   if (!self->no_python_timestamp_fix)
     post_process_flags |= BUILDER_POST_PROCESS_FLAGS_PYTHON_TIMESTAMPS;
 
@@ -1793,6 +1894,7 @@ builder_module_checksum (BuilderModule  *self,
   builder_cache_checksum_strv (cache, self->build_commands);
   builder_cache_checksum_str (cache, self->buildsystem);
   builder_cache_checksum_str (cache, self->install_rule);
+  builder_cache_checksum_compat_boolean (cache, self->run_tests);
 
   if (self->build_options)
     builder_options_checksum (self->build_options, cache, context);

src/builder-options.c

@@ -49,6 +49,7 @@ struct BuilderOptions
   char       *prefix;
   char      **env;
   char      **build_args;
+  char      **test_args;
   char      **config_opts;
   char      **make_args;
   char      **make_install_args;
@@ -78,6 +79,7 @@ enum {
   PROP_NO_DEBUGINFO_COMPRESSION,
   PROP_ARCH,
   PROP_BUILD_ARGS,
+  PROP_TEST_ARGS,
   PROP_CONFIG_OPTS,
   PROP_MAKE_ARGS,
   PROP_MAKE_INSTALL_ARGS,
@@ -103,6 +105,7 @@ builder_options_finalize (GObject *object)
   g_free (self->prefix);
   g_strfreev (self->env);
   g_strfreev (self->build_args);
+  g_strfreev (self->test_args);
   g_strfreev (self->config_opts);
   g_strfreev (self->make_args);
   g_strfreev (self->make_install_args);
@@ -165,6 +168,10 @@ builder_options_get_property (GObject    *object,
       g_value_set_boxed (value, self->build_args);
       break;
 
+    case PROP_TEST_ARGS:
+      g_value_set_boxed (value, self->test_args);
+      break;
+
     case PROP_CONFIG_OPTS:
       g_value_set_boxed (value, self->config_opts);
       break;
@@ -263,6 +270,12 @@ builder_options_set_property (GObject      *object,
       g_strfreev (tmp);
       break;
 
+    case PROP_TEST_ARGS:
+      tmp = self->test_args;
+      self->test_args = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
     case PROP_CONFIG_OPTS:
       tmp = self->config_opts;
       self->config_opts = g_strdupv (g_value_get_boxed (value));
@@ -384,6 +397,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                        "",
                                                        G_TYPE_STRV,
                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_TEST_ARGS,
+                                   g_param_spec_boxed ("test-args",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_CONFIG_OPTS,
                                    g_param_spec_boxed ("config-opts",
@@ -942,6 +962,44 @@ builder_options_get_build_args (BuilderOptions *self,
   return (char **) g_ptr_array_free (g_steal_pointer (&array), FALSE);
 }
 
+char **
+builder_options_get_test_args (BuilderOptions *self,
+                               BuilderContext *context,
+                               GError **error)
+{
+  g_autoptr(GList) options = get_all_options (self, context);
+  GList *l;
+  int i;
+  g_autoptr(GPtrArray) array = g_ptr_array_new_with_free_func (g_free);
+
+  /* Last argument wins, so reverse the list for per-module to win */
+  options = g_list_reverse (options);
+
+  /* Always run tests readonly */
+  g_ptr_array_add (array, g_strdup ("--readonly"));
+
+  for (l = options; l != NULL; l = l->next)
+    {
+      BuilderOptions *o = l->data;
+
+      if (o->test_args)
+        {
+          for (i = 0; o->test_args[i] != NULL; i++)
+            g_ptr_array_add (array, g_strdup (o->test_args[i]));
+        }
+    }
+
+  if (array->len > 0 && builder_context_get_sandboxed (context))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't specify test-args in sandboxed build");
+      return NULL;
+    }
+
+  g_ptr_array_add (array, NULL);
+
+  return (char **) g_ptr_array_free (g_steal_pointer (&array), FALSE);
+}
+
 static char **
 builder_options_get_strv (BuilderOptions *self,
                           BuilderContext *context,
@@ -1019,6 +1077,7 @@ builder_options_checksum (BuilderOptions *self,
   builder_cache_checksum_str (cache, self->prefix);
   builder_cache_checksum_strv (cache, self->env);
   builder_cache_checksum_strv (cache, self->build_args);
+  builder_cache_checksum_compat_strv (cache, self->test_args);
   builder_cache_checksum_strv (cache, self->config_opts);
   builder_cache_checksum_strv (cache, self->make_args);
   builder_cache_checksum_strv (cache, self->make_install_args);

src/builder-options.h

@@ -52,6 +52,9 @@ char **     builder_options_get_env (BuilderOptions *self,
 char **     builder_options_get_build_args (BuilderOptions *self,
                                             BuilderContext *context,
                                             GError **error);
+char **     builder_options_get_test_args (BuilderOptions *self,
+                                            BuilderContext *context,
+                                            GError **error);
 char **     builder_options_get_config_opts (BuilderOptions *self,
                                              BuilderContext *context,
                                              char          **base_opts);