From e24c2218f1e78efac5f091ffa2b328a45fdd721d Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez Date: Sat, 14 Apr 2018 12:17:59 -0500 Subject: [PATCH] Add YAML support as an alternative to JSON (closes #2) Closes: #127 Approved by: alexlarsson --- configure.ac | 11 ++++ src/Makefile.am.inc | 4 +- src/builder-main.c | 22 +++---- src/builder-manifest.c | 8 +-- src/builder-module.c | 8 +-- src/builder-utils.c | 143 +++++++++++++++++++++++++++++++++++++++++ src/builder-utils.h | 8 +++ tests/module1.yaml | 14 ++++ tests/module2.yaml | 11 ++++ tests/test-builder.sh | 6 ++ tests/test.yaml | 52 +++++++++++++++ 11 files changed, 266 insertions(+), 21 deletions(-) create mode 100644 tests/module1.yaml create mode 100644 tests/module2.yaml create mode 100644 tests/test.yaml diff --git a/configure.ac b/configure.ac index 196298c2..2e34e6bc 100644 --- a/configure.ac +++ b/configure.ac @@ -146,6 +146,17 @@ AS_IF([test "x$with_dwarf_header" = "xyes"], AS_IF([test "x$ac_cv_header_dwarf_h" != "xyes"], [AC_MSG_ERROR([dwarf.h is required but was not found])])]) +AC_ARG_WITH([yaml], + [AS_HELP_STRING([--without-yaml], + [Disable YAML support [default=auto]])]) +AS_IF([test "x$with_yaml" != "xno"],[ + PKG_CHECK_MODULES(YAML, [yaml-0.1], [have_yaml=yes], [have_yaml=no]) +], [have_yaml=no]) +AS_IF([test "x$have_yaml" = "xno"],[ + AS_IF([test "x$with_yaml" = "xyes"], + [AC_MSG_ERROR([yaml-0.1 was not found, which is needed for --with-yaml])]) +], [AC_DEFINE([FLATPAK_BUILDER_ENABLE_YAML],[1],[Define if yaml supported])]) + # Do we enable building peer to peer support using libostree’s experimental (non-stable) API? # If so, OSTREE_ENABLE_EXPERIMENTAL_API needs to be #defined before ostree.h is # included. diff --git a/src/Makefile.am.inc b/src/Makefile.am.inc index 7b11606d..e37db576 100644 --- a/src/Makefile.am.inc +++ b/src/Makefile.am.inc @@ -44,5 +44,5 @@ flatpak_builder_SOURCES = \ src/builder-git.h \ $(NULL) -flatpak_builder_LDADD = $(AM_LDADD) $(BASE_LIBS) $(LIBELF_LIBS) libglnx.la -flatpak_builder_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS) +flatpak_builder_LDADD = $(AM_LDADD) $(BASE_LIBS) $(LIBELF_LIBS) $(YAML_LIBS) libglnx.la +flatpak_builder_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS) $(YAML_CFLAGS) diff --git a/src/builder-main.c b/src/builder-main.c index 4428253f..7d2911c3 100644 --- a/src/builder-main.c +++ b/src/builder-main.c @@ -312,9 +312,9 @@ main (int argc, g_autoptr(BuilderManifest) manifest = NULL; g_autoptr(GOptionContext) context = NULL; const char *app_dir_path = NULL, *manifest_rel_path; - g_autofree gchar *json = NULL; - g_autofree gchar *json_sha256 = NULL; - g_autofree gchar *old_json_sha256 = NULL; + g_autofree gchar *manifest_contents = NULL; + g_autofree gchar *manifest_sha256 = NULL; + g_autofree gchar *old_manifest_sha256 = NULL; g_autoptr(BuilderContext) build_context = NULL; g_autoptr(GFile) base_dir = NULL; g_autoptr(GFile) manifest_file = NULL; @@ -536,18 +536,18 @@ main (int argc, builder_context_set_base_dir (build_context, base_dir); - if (!g_file_get_contents (flatpak_file_get_path_cached (manifest_file), &json, NULL, &error)) + if (!g_file_get_contents (flatpak_file_get_path_cached (manifest_file), &manifest_contents, NULL, &error)) { g_printerr ("Can't load '%s': %s\n", manifest_rel_path, error->message); return 1; } - json_sha256 = g_compute_checksum_for_string (G_CHECKSUM_SHA256, json, -1); + manifest_sha256 = g_compute_checksum_for_string (G_CHECKSUM_SHA256, manifest_contents, -1); if (opt_skip_if_unchanged) { - old_json_sha256 = builder_context_get_checksum_for (build_context, manifest_basename); - if (old_json_sha256 != NULL && strcmp (json_sha256, old_json_sha256) == 0) + old_manifest_sha256 = builder_context_get_checksum_for (build_context, manifest_basename); + if (old_manifest_sha256 != NULL && strcmp (manifest_sha256, old_manifest_sha256) == 0) { g_print ("No changes to manifest, skipping\n"); return 42; @@ -557,8 +557,8 @@ main (int argc, /* Can't push this as user data to the demarshalling :/ */ builder_manifest_set_demarshal_base_dir (builder_context_get_base_dir (build_context)); - manifest = (BuilderManifest *) json_gobject_from_data (BUILDER_TYPE_MANIFEST, - json, -1, &error); + manifest = (BuilderManifest *) builder_gobject_from_data (BUILDER_TYPE_MANIFEST, manifest_rel_path, + manifest_contents, &error); builder_manifest_set_demarshal_base_dir (NULL); @@ -682,7 +682,7 @@ main (int argc, } } - if (!builder_context_set_checksum_for (build_context, manifest_basename, json_sha256, &error)) + if (!builder_context_set_checksum_for (build_context, manifest_basename, manifest_sha256, &error)) { g_printerr ("Failed to set checksum for ‘%s’: %s\n", manifest_basename, error->message); return 1; @@ -793,7 +793,7 @@ main (int argc, } if (builder_context_get_bundle_sources (build_context) && - !builder_manifest_bundle_sources (manifest, json, cache, build_context, &error)) + !builder_manifest_bundle_sources (manifest, manifest_contents, cache, build_context, &error)) { g_printerr ("Error: %s\n", error->message); return 1; diff --git a/src/builder-manifest.c b/src/builder-manifest.c index db88be43..4deaa22c 100644 --- a/src/builder-manifest.c +++ b/src/builder-manifest.c @@ -1137,15 +1137,15 @@ builder_manifest_deserialize_property (JsonSerializable *serializable, g_autoptr(GFile) module_file = g_file_resolve_relative_path (demarshal_base_dir, module_relpath); const char *module_path = flatpak_file_get_path_cached (module_file); - g_autofree char *json = NULL; + g_autofree char *module_contents = NULL; g_autoptr(GError) error = NULL; - if (g_file_get_contents (module_path, &json, NULL, &error)) + if (g_file_get_contents (module_path, &module_contents, NULL, &error)) { g_autoptr(GFile) module_file_dir = g_file_get_parent (module_file); builder_manifest_set_demarshal_base_dir (module_file_dir); - module = json_gobject_from_data (BUILDER_TYPE_MODULE, - json, -1, &error); + module = builder_gobject_from_data (BUILDER_TYPE_MODULE, + module_relpath, module_contents, &error); builder_manifest_set_demarshal_base_dir (saved_demarshal_base_dir); if (module) { diff --git a/src/builder-module.c b/src/builder-module.c index 146ed797..a02922fd 100644 --- a/src/builder-module.c +++ b/src/builder-module.c @@ -824,14 +824,14 @@ builder_module_deserialize_property (JsonSerializable *serializable, g_autoptr(GFile) module_file = g_file_resolve_relative_path (saved_demarshal_base_dir, module_relpath); const char *module_path = flatpak_file_get_path_cached (module_file); - g_autofree char *json = NULL; + g_autofree char *module_contents = NULL; - if (g_file_get_contents (module_path, &json, NULL, NULL)) + if (g_file_get_contents (module_path, &module_contents, NULL, NULL)) { g_autoptr(GFile) module_file_dir = g_file_get_parent (module_file); builder_manifest_set_demarshal_base_dir (module_file_dir); - module = json_gobject_from_data (BUILDER_TYPE_MODULE, - json, -1, NULL); + module = builder_gobject_from_data (BUILDER_TYPE_MODULE, + module_relpath, module_contents, NULL); builder_manifest_set_demarshal_base_dir (saved_demarshal_base_dir); if (module) { diff --git a/src/builder-utils.c b/src/builder-utils.c index 0f5475a8..c5f10c97 100644 --- a/src/builder-utils.c +++ b/src/builder-utils.c @@ -39,6 +39,15 @@ #include "builder-flatpak-utils.h" #include "builder-utils.h" +#ifdef FLATPAK_BUILDER_ENABLE_YAML +#include + +G_DEFINE_QUARK (builder-yaml-parse-error, builder_yaml_parse_error) + +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (yaml_parser_t, yaml_parser_delete) +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (yaml_document_t, yaml_document_delete) +#endif + char * builder_uri_to_filename (const char *uri) { @@ -367,6 +376,140 @@ builder_migrate_locale_dirs (GFile *root_dir, return TRUE; } +#ifdef FLATPAK_BUILDER_ENABLE_YAML + +static JsonNode * +parse_yaml_node_to_json (yaml_document_t *doc, yaml_node_t *node) +{ + JsonNode *json = json_node_alloc (); + const char *scalar = NULL; + g_autoptr(JsonArray) array = NULL; + g_autoptr(JsonObject) object = NULL; + yaml_node_item_t *item = NULL; + yaml_node_pair_t *pair = NULL; + + switch (node->type) + { + case YAML_NO_NODE: + json_node_init_null (json); + break; + case YAML_SCALAR_NODE: + scalar = (gchar *) node->data.scalar.value; + if (node->data.scalar.style == YAML_PLAIN_SCALAR_STYLE) + { + if (strcmp (scalar, "true") == 0) + { + json_node_init_boolean (json, TRUE); + break; + } + else if (strcmp (scalar, "false") == 0) + { + json_node_init_boolean (json, FALSE); + break; + } + + gchar *endptr; + gint64 num = g_ascii_strtoll (scalar, &endptr, 10); + if (*scalar != '\0' && *endptr == '\0') + { + json_node_init_int (json, num); + break; + } + } + + json_node_init_string (json, scalar); + break; + case YAML_SEQUENCE_NODE: + array = json_array_new (); + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) + { + yaml_node_t *child = yaml_document_get_node (doc, *item); + if (child != NULL) + json_array_add_element (array, parse_yaml_node_to_json (doc, child)); + } + + json_node_init_array (json, array); + break; + case YAML_MAPPING_NODE: + object = json_object_new (); + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) + { + yaml_node_t *key = yaml_document_get_node (doc, pair->key); + yaml_node_t *value = yaml_document_get_node (doc, pair->value); + + g_warn_if_fail (key->type == YAML_SCALAR_NODE); + json_object_set_member (object, (gchar *) key->data.scalar.value, + parse_yaml_node_to_json (doc, value)); + } + + json_node_init_object (json, object); + break; + } + + return json; +} + +static JsonNode * +parse_yaml_to_json (const gchar *contents, + GError **error) +{ + if (error) + *error = NULL; + + g_auto(yaml_parser_t) parser = {0}; + g_auto(yaml_document_t) doc = {{0}}; + + if (!yaml_parser_initialize (&parser)) + g_error ("yaml_parser_initialize is out of memory."); + yaml_parser_set_input_string (&parser, (yaml_char_t *) contents, strlen (contents)); + + if (!yaml_parser_load (&parser, &doc)) + { + g_set_error (error, BUILDER_YAML_PARSE_ERROR, parser.error, "%zu:%zu: %s", parser.problem_mark.line + 1, + parser.problem_mark.column + 1, parser.problem); + return NULL; + } + + yaml_node_t *root = yaml_document_get_root_node (&doc); + if (root == NULL) + { + g_set_error (error, BUILDER_YAML_PARSE_ERROR, YAML_PARSER_ERROR, + "Document has no root node."); + return NULL; + } + + return parse_yaml_node_to_json (&doc, root); +} + +#else // FLATPAK_BUILDER_ENABLE_YAML + +static JsonNode * +parse_yaml_to_json (const gchar *contents, + GError **error) +{ + g_set_error (error, BUILDER_YAML_PARSE_ERROR, 0, "flatpak-builder was not compiled with YAML support."); + return NULL; +} + +#endif // FLATPAK_BUILDER_ENABLE_YAML + +GObject * +builder_gobject_from_data (GType gtype, + const char *relpath, + const char *contents, + GError **error) +{ + if (g_str_has_suffix (relpath, ".yaml") || g_str_has_suffix (relpath, ".yml")) + { + g_autoptr(JsonNode) json = parse_yaml_to_json (contents, error); + if (json != NULL) + return json_gobject_deserialize (gtype, json); + else + return NULL; + } + else + return json_gobject_from_data (gtype, contents, -1, error); +} /* * This code is based on debugedit.c from rpm, which has this copyright: diff --git a/src/builder-utils.h b/src/builder-utils.h index e098d4c3..d1b8ad38 100644 --- a/src/builder-utils.h +++ b/src/builder-utils.h @@ -61,6 +61,14 @@ void flatpak_collect_matches_for_path_pattern (const char *path, gboolean builder_migrate_locale_dirs (GFile *root_dir, GError **error); +GQuark builder_yaml_parse_error_quark (void); +#define BUILDER_YAML_PARSE_ERROR (builder_yaml_parse_error_quark ()) + +GObject * builder_gobject_from_data (GType gtype, + const char *relpath, + const char *contents, + GError **error); + gboolean builder_host_spawnv (GFile *dir, char **output, GSubprocessFlags flags, diff --git a/tests/module1.yaml b/tests/module1.yaml new file mode 100644 index 00000000..0bfb20dd --- /dev/null +++ b/tests/module1.yaml @@ -0,0 +1,14 @@ +name: module1 +modules: + - name: module1-first + buildsystem: simple + build-commands: + - 'echo module1 > /app/ran_module1' + - 'echo module1 > /app/modify_me' + sources: + - type: file + path: data1 + - type: patch + strip-components: 0 + path: data1.patch + - include2/module2.yaml diff --git a/tests/module2.yaml b/tests/module2.yaml new file mode 100644 index 00000000..ee35af6a --- /dev/null +++ b/tests/module2.yaml @@ -0,0 +1,11 @@ +name: module2 +buildsystem: simple +ensure-writable: [ /modify_me ] +build-commands: + - 'echo module2 > /app/ran_module2' + - 'echo module2 > /app/modify_me' +sources: + - type: file + path: data2 + - type: patch + path: data2.patch diff --git a/tests/test-builder.sh b/tests/test-builder.sh index bdeb3b95..396fc183 100755 --- a/tests/test-builder.sh +++ b/tests/test-builder.sh @@ -37,17 +37,21 @@ cd $TEST_DATA_DIR/ cp -a $(dirname $0)/test-configure . echo "version1" > app-data cp $(dirname $0)/test.json . +cp $(dirname $0)/test.yaml . cp $(dirname $0)/test-runtime.json . cp $(dirname $0)/0001-Add-test-logo.patch . mkdir include1 cp $(dirname $0)/module1.json include1/ +cp $(dirname $0)/module1.yaml include1/ cp $(dirname $0)/data1 include1/ cp $(dirname $0)/data1.patch include1/ mkdir include1/include2 cp $(dirname $0)/module2.json include1/include2/ +cp $(dirname $0)/module2.yaml include1/include2/ cp $(dirname $0)/data2 include1/include2/ cp $(dirname $0)/data2.patch include1/include2/ ${FLATPAK_BUILDER} --repo=$REPO $FL_GPGARGS --force-clean appdir test.json +${FLATPAK_BUILDER} --repo=$REPO $FL_GPGARGS --force-clean appdir test.yaml assert_file_has_content appdir/files/share/app-data version1 assert_file_has_content appdir/metadata shared=network; @@ -81,6 +85,8 @@ echo "ok install+run" echo "version2" > app-data ${FLATPAK_BUILDER} $FL_GPGARGS --repo=$REPO --force-clean appdir test.json assert_file_has_content appdir/files/share/app-data version2 +${FLATPAK_BUILDER} $FL_GPGARGS --repo=$REPO --force-clean appdir test.yaml +assert_file_has_content appdir/files/share/app-data version2 ${FLATPAK} ${U} update org.test.Hello2 master diff --git a/tests/test.yaml b/tests/test.yaml new file mode 100644 index 00000000..a210510e --- /dev/null +++ b/tests/test.yaml @@ -0,0 +1,52 @@ +app-id: org.test.Hello2 +runtime: org.test.Platform +sdk: org.test.Sdk +command: hello2.sh +tags: [test] +finish-args: + - --share=network +build-options: + cflags: -O2 -g + cxxflags: -O2 -g + env: + FOO: bar + V: '1' +cleanup: [/cleanup, '*.cleanup'] +cleanup-commands: [touch /app/cleaned_up] +modules: + - include1/module1.yaml + - name: root + modules: + - name: test + config-opts: [--some-arg] + post-install: + - touch /app/bin/file.cleanup + - mkdir -p /app/share/icons/ + - cp org.test.Hello.png /app/share/icons/ + make-args: [BAR=2] + make-install-args: [BAR=3] + build-commands: ['echo foo > /app/out'] + sources: + - type: file + path: test-configure + dest-filename: configure + sha256: 675a1ac2feec4d4f54e581b4b01bc3cfd2c1cf31aa5963574d31228c8a11b7e7 + - type: file + path: app-data + - type: script + dest-filename: hello2.sh + commands: ['echo "Hello world2, from a sandbox"'] + - type: shell + commands: + - mkdir /app/cleanup/ + - touch /app/cleanup/a_file + - type: patch + path: 0001-Add-test-logo.patch + use-git: true + - name: test2 + build-commands: ['echo foo2 > /app/out2'] + buildsystem: simple + sources: + - type: file + path: app-data + - name: empty