Merge pull request #333 from smcv/brz

bzr: Run brz in preference to bzr

.github/issue_template.md

@@ -0,0 +1,5 @@
+## Linux distribution and version
+## Flatpak-builder version
+## Flatpak version
+## Description of the problem
+## Steps to reproduce

.github/workflows/block-autosquash-commits.yml

@@ -0,0 +1,15 @@
+on: pull_request
+
+name: Pull Requests
+
+jobs:
+  message-check:
+    name: Block Autosquash Commits
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Block Autosquash Commits
+        uses: xt0rted/block-autosquash-commits-action@v2.0.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/check.yml

@@ -0,0 +1,69 @@
+name: Flatpak-builder CI
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  check:
+    name: Build with gcc and test
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Install Dependencies
+      run: |
+        sudo add-apt-repository ppa:alexlarsson/flatpak
+        sudo add-apt-repository ppa:alexlarsson/glib260
+        sudo apt-get update
+        sudo apt-get install -y libglib2.0 attr automake gettext autopoint bison  dbus gtk-doc-tools \
+        libfuse-dev ostree libostree-dev libarchive-dev libcap-dev libattr1-dev libdw-dev libelf-dev \
+        libjson-glib-dev shared-mime-info desktop-file-utils libpolkit-agent-1-dev libpolkit-gobject-1-dev \
+        libseccomp-dev libsoup2.4-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
+        libgirepository1.0-dev libappstream-glib-dev libdconf-dev clang socat flatpak \
+        libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch unzip
+    - name: Check out flatpak
+      uses: actions/checkout@v1
+      with:
+        submodules: true
+    - name: configure
+      # TODO: Enable gtk-doc builds
+      run: ./autogen.sh
+      env:
+        CFLAGS: -fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2
+    - name: Build flatpak
+      run: make -j $(getconf _NPROCESSORS_ONLN)
+    - name: Run tests
+      run: make check
+      env:
+        ASAN_OPTIONS: detect_leaks=0 # Right now we're not fully clean, but this gets us use-after-free etc
+
+  clang:
+    name: Build with clang
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Install Dependencies
+      run: |
+        sudo add-apt-repository ppa:alexlarsson/flatpak
+        sudo add-apt-repository ppa:alexlarsson/glib260
+        sudo apt-get update
+        sudo apt-get install -y libglib2.0 attr automake gettext autopoint bison  dbus gtk-doc-tools \
+        libfuse-dev ostree libostree-dev libarchive-dev libcap-dev libattr1-dev libdw-dev libelf-dev \
+        libjson-glib-dev shared-mime-info desktop-file-utils libpolkit-agent-1-dev libpolkit-gobject-1-dev \
+        libseccomp-dev libsoup2.4-dev libsystemd-dev libxml2-utils libgpgme11-dev gobject-introspection \
+        libgirepository1.0-dev libappstream-glib-dev libdconf-dev clang flatpak \
+        libcurl4-gnutls-dev libflatpak-dev libyaml-dev elfutils git patch unzip
+    - name: Check out flatpak
+      uses: actions/checkout@v1
+      with:
+        submodules: true
+    - name: configure
+      # We disable introspection because it fails with clang: https://bugzilla.redhat.com/show_bug.cgi?id=1543295
+      run: ./autogen.sh --disable-introspection
+      env:
+        CC: clang
+        CFLAGS: -Werror=unused-variable
+    - name: Build flatpak
+      run: make -j $(getconf _NPROCESSORS_ONLN)

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "libglnx"]
 	path = libglnx
-	url = https://git.gnome.org/browse/libglnx
+	url = https://gitlab.gnome.org/GNOME/libglnx.git

.papr.yml

@@ -6,13 +6,13 @@ branches:
     - try
 
 required: true
-context: f26-primary
+context: f28-primary
 
 # This test case wants an "unprivileged container with bubblewrap",
 # which we don't have right now; so just provision a VM and do a
 # docker --privileged run.
 host:
-  distro: fedora/26/atomic
+  distro: fedora/28/atomic
 
 env:
     # TODO: CFLAGS: Readd -fsanitize-undefined-trap-on-error -fsanitize=address after debugging
@@ -29,7 +29,7 @@ tests:
     -e "ASAN_OPTIONS=${ASAN_OPTIONS:-}"
     -v /etc/yum.repos.d:/etc/yum.repos.d.host:ro
     -v $(pwd):/srv/code -w /srv/code
-    registry.fedoraproject.org/fedora:26 /bin/sh -c
+    registry.fedoraproject.org/fedora:28 /bin/sh -c
     "cp -fv /etc/yum.repos.d{.host/*.repo,} &&
      ./ci/build-check.sh"
 

NEWS

@@ -1,3 +1,151 @@
+Changes in 1.0.10
+=================
+
+ * Support for 7zip archives
+ * Allow overriding archive type
+ * Support --token-type option
+
+Changes in 1.0.9
+================
+
+ * More detailed commandline output with --verbose enabled
+ * Support for including yaml module snippets
+ * Support the CCACHE_DIR environment variable
+ * New option disable-submodules for git sources
+ * New commandline option --show-manifest
+ * Multiple modules with the same name in the same manifest is
+   now supported.
+
+Changes in 1.0.8
+================
+
+ * Support multiple instances of --install-dep-from
+ * Ensure shallow git mirrors are updated when the git ref changes
+ * Improved error reporting
+
+Changes in 1.0.7
+================
+
+ * Use the new die-with-parent feature of HostCommand when build to run in a flatpak
+ * Fix some details in how we create platform commits to fix font cache mtime issues.
+
+Changes in 1.0.6
+================
+
+ * Pass --noninteractive when installing for flatpak >= 1.2.0
+ * Doc fixes
+
+Changes in 1.0.5
+================
+
+Due to a change in the output of "flatpak info" the support for
+loading default build options from the sdk regressed with flatpak
+1.2.x. This is fixed and everyone who built flatpaks with 1.2.x are
+recommended to update to this release and re-build.
+
+Changes in 1.0.4
+================
+
+ * Add --add/remove-tag options
+ * Allow using a full ref as sdk and use the branch from that.
+
+Changes in 1.0.3
+================
+
+ * Compress changes in cache, fixing the build of large applications like libreoffice in
+   some cases.
+ * Add new default-branch manifest option to allow overridable branch name.
+
+Changes in 1.0.2
+================
+
+ * Print a warning when a YAML document has a float-like string value.
+ * Handle .txz archive suffixes.
+ * Set SOURCE_DATE_EPOCH to the mtime of the manifest in the build.
+ * Update python post-processor to handle python 3.7 pyc files.
+ * Allow unsetting env vars in the build options by setting them to null.
+ * Documentation improvements
+
+Changes in 1.0.1
+================
+
+ * Run appstream-util mirror-screenshots in the sandbox, rather than on
+   the host, as the host might not have a new enough appstream-glib.
+ * Fix build cache when a module has uncommon characters in the name.
+
+Changes in 1.0.0
+================
+
+ * Respect no-parallel-make with ninja
+ * Fixed regression in data: uris
+ * Warn about deprecated cmake property
+ * Make X-Flatpak-RenamedFrom a list
+ * Fix handling of mirror uris with no terminating slash
+ * Take arch into account for --skip-if-unchanged
+ * Fix --run when run inside a flatpak sandbox
+ * Ensure GIT commiter env var is set when applying git patches
+ * Keep x-* properties in the manifest when creating manifest.json
+ * New property `paths` in patch source to apply multiple patches.
+
+Major changes in 0.99.3
+=======================
+
+ * Add X-Flatpak-RenamedFrom key to renamed desktop files
+ * Set FLATPAK_BUILDER_BUILDDIR when building module
+ * Handle large downloads better
+ * Make p2p code non-optional
+
+Major changes in 0.99.2
+=======================
+
+ * Fix --install not working due to asking for confirmation
+ * Fix builds with latest libsoup
+ * New option git-init for archives
+ * New option use-git-am for patches
+
+Major changes in 0.99.1
+=======================
+
+In preparation for 1.0, this release depends on a recent (0.11.8) flatpak.
+
+ * New libdir build option
+ * Support loading default options from sdk
+ * Add support for overriding c/cpp/ld/etc flags in a module
+ * Now uses libcurl to download things instead of soup,
+   which means we now support ftp urls.
+ * Don't prune build caches when building with --{export,finish}-only
+ * rename-desktop-file now handles launchables in the appdata
+ * archive sources support dest-filename
+ * archive and file sources now support mirror-urls to list
+   extra urls in case of network problems.
+
+Major changes in 0.10.10
+========================
+
+ * We now support yaml manifest as well as json.
+ * New source type "svn" to download subversion checkouts.
+ * New source type "dir" for local directories.
+ * Set FLATPAK_BUILDER_N_JOBS in environment during build.
+ * If not using ccache, enfore it of. This helps builds with
+   meson as they pick up ccache by default, which makes no
+   sense when there is no persistent storage for ccache to use.
+ * New options prepend-*-path similar to the existing append-*-path
+   ones, but prepending instead.
+ * New argument --no-shallow-clone which disables any shallow clone
+   optimizations.
+ * New inherit-sdk-extensions property similar to inherit-extensions,
+   but is not inherited in the platform.
+ * We now ignore json keys starting with "//" and "__" allowing these
+   to be used for comments.
+ * Fixed a bug that caused --install to not work properly.
+ * New argument --extension-tag for using the new tagged extension feature.
+ * The build metadata now records which extensions were produced as part
+   of a build.
+ * New property add-build-extensions allows creating exension points that
+   can be used during the build.
+ * flatpak-builder --run now also binds /run/build-runtime to the builds,
+   allowing it to work when building runtimes.
+
 Major changes in 0.10.9
 =======================
 

ci/build.sh

@@ -8,16 +8,17 @@ dn=$(dirname $0)
 
 pkg_install sudo which attr fuse \
     libubsan libasan libtsan elfutils-libelf-devel libdwarf-devel \
-    elfutils git gettext-devel libappstream-glib-devel \
+    elfutils git gettext-devel libappstream-glib-devel bison \
+    libcurl-devel dconf-devel fuse-devel \
     /usr/bin/{update-mime-database,update-desktop-database,gtk-update-icon-cache}
-pkg_install_testing ostree-devel ostree
-pkg_install_if_os fedora gjs parallel clang
+pkg_install_testing ostree-devel ostree libyaml-devel
+pkg_install_if_os fedora gjs parallel clang python2
 pkg_install_builddeps flatpak
 
 (git clone --depth=1 https://github.com/flatpak/flatpak/
  cd flatpak
  unset CFLAGS # the sanitizers require calling apps be linked too
- build
+ build --disable-introspection
  make install
  flatpak --version
 )

ci/libbuild.sh

@@ -55,5 +55,6 @@ pkg_install_builddeps() {
     # builddeps+runtime deps
     pkg_builddep $pkg
     pkg_install $pkg
-    rpm -e $pkg
+    yum -y update gcc gcc-c++ annobin # This had some weird conflict with gcc
+    rpm -e --nodeps $pkg
 }

configure.ac

@@ -13,9 +13,9 @@ AC_PREREQ([2.63])
 # on the stable branch, interface age == micro
 # on the unstable (ie master), interface age = 0
 
-m4_define([flatpak_builder_major_version], [0])
-m4_define([flatpak_builder_minor_version], [10])
-m4_define([flatpak_builder_micro_version], [9])
+m4_define([flatpak_builder_major_version], [1])
+m4_define([flatpak_builder_minor_version], [0])
+m4_define([flatpak_builder_micro_version], [10])
 m4_define([flatpak_builder_version],
           [flatpak_builder_major_version.flatpak_builder_minor_version.flatpak_builder_micro_version])
 
@@ -27,7 +27,7 @@ AC_INIT([flatpak-builder],
 
 GLIB_REQS=2.44
 OSTREE_REQS=2017.14
-FLATPAK_REQS=0.10.2
+FLATPAK_REQS=0.99.1
 
 AC_USE_SYSTEM_EXTENSIONS
 AC_SYS_LARGEFILE
@@ -35,6 +35,7 @@ LIBGLNX_CONFIGURE
 
 AC_PROG_CC
 AM_PROG_CC_C_O
+AC_PROG_CC_STDC
 AC_DISABLE_STATIC
 
 AC_CHECK_PROG([FLATPAK], [flatpak], [flatpak], [false])
@@ -94,7 +95,7 @@ AC_CHECK_FUNCS(fdwalk)
 AC_CHECK_HEADER([sys/xattr.h], [], [AC_MSG_ERROR([You must have sys/xattr.h from glibc])])
 AC_CHECK_HEADER([sys/capability.h], have_caps=yes, [AC_MSG_ERROR([sys/capability.h header not found])])
 
-PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4])
+PKG_CHECK_MODULES(BASE, [glib-2.0 >= $GLIB_REQS gio-2.0 gio-unix-2.0 libsoup-2.4 ostree-1 >= $OSTREE_REQS json-glib-1.0 libxml-2.0 >= 2.4 libcurl])
 
 dnl ************************
 dnl *** check for libelf ***
@@ -146,27 +147,16 @@ AS_IF([test "x$with_dwarf_header" = "xyes"],
        AS_IF([test "x$ac_cv_header_dwarf_h" != "xyes"],
              [AC_MSG_ERROR([dwarf.h is required but was not found])])])
 
-# Do we enable building peer to peer support using libostree’s experimental (non-stable) API?
-# If so, OSTREE_ENABLE_EXPERIMENTAL_API needs to be #defined before ostree.h is
-# included.
-AC_ARG_ENABLE([p2p],
-  [AS_HELP_STRING([--enable-p2p],
-                  [Enable unstable peer to peer support [default=no]])],,
-  [enable_p2p=no])
-AS_IF([test x$enable_p2p = xyes],[
-  PKG_CHECK_MODULES(OSTREE, [ostree-1 >= $OSTREE_REQS])
-
-  ostree_features=$($PKG_CONFIG --variable=features ostree-1)
-  AS_CASE(["$ostree_features"],
-          [*experimental*],[have_ostree_experimental=yes])
-
-  AS_IF([test "x$have_ostree_experimental" != "xyes"],
-        [AC_MSG_ERROR([Experimental API not found in ostree-1, which is needed for --enable-p2p. OSTree must be compiled with --enable-experimental-api.])])
-
-  AC_DEFINE([OSTREE_ENABLE_EXPERIMENTAL_API],[1],[Define if libostree experimental API should be enabled])
-  AC_DEFINE([FLATPAK_ENABLE_P2P],[1],[Define if peer to peer support should be enabled])
-])
-AM_CONDITIONAL([ENABLE_P2P],[test x$enable_p2p = xyes])
+AC_ARG_WITH([yaml],
+  [AS_HELP_STRING([--without-yaml],
+                  [Disable YAML support [default=auto]])])
+AS_IF([test "x$with_yaml" != "xno"],[
+  PKG_CHECK_MODULES(YAML, [yaml-0.1], [have_yaml=yes], [have_yaml=no])
+], [have_yaml=no])
+AS_IF([test "x$have_yaml" = "xno"],[
+  AS_IF([test "x$with_yaml" = "xyes"],
+        [AC_MSG_ERROR([yaml-0.1 was not found, which is needed for --with-yaml])])
+], [AC_DEFINE([FLATPAK_BUILDER_ENABLE_YAML],[1],[Define if yaml supported])])
 
 AC_ARG_ENABLE(documentation,
               AC_HELP_STRING([--enable-documentation], [Build documentation]),,

doc/flatpak-builder-docs.xml.in

@@ -8,6 +8,11 @@
       <releaseinfo>Version @VERSION@</releaseinfo>
     </referenceinfo>
     <title>Flatpak Builder Command Reference</title>
+    <important>
+        <para>
+          The command reference is generated from the flatpak-builder repo; see <ulink url="https://github.com/flatpak/flatpak-builder/tree/master/doc"></ulink>
+        </para>
+    </important>
     <partintro>
       <para>
         Flatpak-builder is a tool to build flatpak applications.

doc/flatpak-builder.xml

@@ -49,6 +49,12 @@
                 <arg choice="opt" rep="repeat">OPTION</arg>
                 <arg choice="plain">MANIFEST</arg>
             </cmdsynopsis>
+            <cmdsynopsis>
+                <command>flatpak-builder</command>
+                <arg choice="plain">--show-manifest</arg>
+                <arg choice="opt" rep="repeat">OPTION</arg>
+                <arg choice="plain">MANIFEST</arg>
+            </cmdsynopsis>
     </refsynopsisdiv>
 
     <refsect1>
@@ -99,22 +105,20 @@
             new commits added, or the first module where some changes to the <arg choice="plain">MANIFEST</arg> file caused
             the build environment to change. This makes flatpak-builder very efficient for incremental builds.
         </para>
-        <!-- FIXME: Uncomment this when enable-p2p is enabled unconditionally.
         <para>
             When building a flatpak to be published to the internet,
-            <option>-FIXME-collection-id=COLLECTION-ID</option> should be specified
+            <option>--collection-id=COLLECTION-ID</option> should be specified
             as a globally unique reverse DNS value to identify the collection of
             flatpaks this will be added to. Setting a globally unique collection
             ID allows the apps in the repository to be shared over peer to peer
             systems without needing further configuration.
         </para>
-        -->
     </refsect1>
 
     <refsect1>
         <title>Manifest</title>
 
-        <para>The manifest file is a json file whose format is described in detail in its own manual page.</para>
+        <para>The manifest file is a json or yaml file whose format is described in detail in its own manual page.</para>
     </refsect1>
 
     <refsect1>
@@ -199,7 +203,7 @@
 
                 <listitem><para>
                   Download missing sources, but don't update local mirrors of version control repos. This is useful
-                  to rebuild things but without updating git or bzr repositories from the remote repository.
+                  to rebuild things but without updating git, bzr or svn repositories from the remote repository.
                 </para></listitem>
             </varlistentry>
 
@@ -217,7 +221,7 @@
                 <listitem><para>
                   Run a command in a sandbox based on the build dir. This starts flatpak build, with some extra
                   arguments to give the same environment as the build, and the same permissions the final app
-                  will have. The command to run must be the last argument passed to
+                  will have (except filesystem permissions). The command to run must be the last argument passed to
                   flatpak-builder, after the directory and the manifest.
                   </para>
 
@@ -255,6 +259,23 @@
                 </listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--show-manifest</option></term>
+
+                <listitem><para>
+                  Loads the manifest, including any included files and prints it in a canonical json format.
+                  This is useful for tools that want to handle manifest files to avoid having to support both
+                  yaml and json, as well as some non-standard json handling that is supported (for example
+                  comments and multiline strings).
+                  </para>
+
+                  <para>
+                  Only the <option>--verbose</option> option can be combined
+                  with this option.
+                  </para>
+                </listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--download-only</option></term>
 
@@ -346,7 +367,8 @@
                 <term><option>--ccache</option></term>
 
                 <listitem><para>
-                     Enable use of ccache in the build (needs ccache in the sdk)
+                     Enable use of ccache in the build (needs ccache in the sdk). The default ccache folder can be
+                     overridden by setting the environment variable CCACHE_DIR.
                 </para></listitem>
             </varlistentry>
 
@@ -366,7 +388,10 @@
                 <term><option>--repo=DIR</option></term>
 
                 <listitem><para>
-                    When build is done, run export the result to this repository.
+                    After the build finishes, run <command>flatpak build-export</command> to
+                    export the result to the repository <arg choice="plain">DIR</arg>. If
+                    <arg choice="plain">DIR</arg> exists, it must be an OSTree repository;
+                    otherwise a new one will be created.
                 </para></listitem>
             </varlistentry>
 
@@ -390,9 +415,8 @@
                 </para></listitem>
             </varlistentry>
 
-            <!-- FIXME: Uncomment this when enable-p2p is enabled unconditionally.
             <varlistentry>
-                <term><option>-FIXME-collection-id=COLLECTION-ID</option></term>
+                <term><option>--collection-id=COLLECTION-ID</option></term>
 
                 <listitem><para>
                     Set as the collection ID of the repository. Setting a globally
@@ -403,7 +427,19 @@
                     repository.
                 </para></listitem>
             </varlistentry>
-            -->
+
+            <varlistentry>
+                <term><option>--token-type=VAL</option></term>
+
+                <listitem><para>
+                    Set type of token needed to install this commit.
+                    Setting this to a value greater than 0 implies that
+                    authentication will be needed to install the
+                    flatpak. A <option>token-type</option> property set
+                    in the manifest takes precedence over this option.
+                    Used when exporting the build results.
+                </para></listitem>
+            </varlistentry>
 
             <varlistentry>
                 <term><option>--gpg-sign=KEYID</option></term>
@@ -499,7 +535,7 @@
                 <term><option>--extra-sources=SOURCE-DIR</option></term>
 
                 <listitem><para>
-                  When downloading sources (archives, files, git, bzr), look in this
+                  When downloading sources (archives, files, git, bzr, svn), look in this
                   directory for pre-existing copies and use them instead of downloading.
                 </para></listitem>
             </varlistentry>
@@ -508,7 +544,7 @@
                 <term><option>--extra-sources-url=URL</option></term>
 
                 <listitem><para>
-                  When downloading sources (archives, files, git, bzr), look at this url
+                  When downloading sources (archives, files, git, bzr, svn), look at this url
                   for mirrored downloads before downloading from the original url.
                 </para></listitem>
             </varlistentry>
@@ -538,6 +574,24 @@
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--add-tag=TAG</option></term>
+
+                <listitem><para>
+                  Add this tag to the tags list of the manifest before building.
+                </para></listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term><option>--remove-tag=TAG</option></term>
+
+                <listitem><para>
+                  Remove this tag to the tags list of the manifest before building. The remove
+                  happen before processing the --add-tag option, so if both are specified, then
+                  --app-tag wins.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--install-deps-from=REMOTE</option></term>
 
@@ -627,7 +681,7 @@
     "runtime-version": "1.2",
     "sdk": "org.freedesktop.Sdk",
     "command": "test",
-    "clean": [ "/include", "*.la" ],
+    "cleanup": [ "/include", "*.la" ],
     "build-options" : {
         "cflags": "-O2 -g",
         "cxxflags": "-O2 -g",
@@ -668,7 +722,7 @@
             "sources": [
                 {
                     "type": "git",
-                    "url": "git://git.gnome.org/babl"
+                    "url": "https://gitlab.gnome.org/GNOME/babl.git"
                 }
             ]
         },

doc/flatpak-manifest.xml

@@ -32,7 +32,7 @@
         <title>Description</title>
 
         <para>
-          Flatpak uses manifest, or recipe, files in a json format to describe how an
+          Flatpak uses manifest, or recipe, files in a json or yaml format to describe how an
           application and its bundled dependencies can be built from sources. The manifest
           gets used by flatpak-builder.
        </para>
@@ -42,7 +42,7 @@
         <title>File format</title>
 
         <para>
-          The top level of the json file describes global attributes of the application, how
+          The top level of the manifest file describes global attributes of the application, how
           it can be built, and the list of modules that need to be built.
         </para>
 
@@ -58,9 +58,25 @@
                 </varlistentry>
                 <varlistentry>
                     <term><option>branch</option> (string)</term>
-                    <listitem><para>The branch of the application, defaults to master.</para></listitem>
+                    <listitem><para>The branch to use when exporting
+                    the application. If this is unset the defaults
+                    come from the default-branch option.</para>
+                    <para>This key overrides both the default-branch
+                    key, and the --default-branch commandline
+                    option. Unless you need a very specific branchname
+                    (like for a runtime or an extension) it is
+                    recommended to use the default-branch key instead, because
+                    you can then override the default using --default-branch when
+                    building for instance a test build.</para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>default-branch</option> (string)</term>
+                    <listitem><para>The default branch to use when
+                    exporting the application. Defaults to master. </para>
+                    <para>This key can be overridden by the
+                    --default-branch commandline
+                    option.</para></listitem>
                 </varlistentry>
-                <!-- FIXME: Uncomment this when enable-p2p is enabled unconditionally.
                 <varlistentry>
                     <term><option>collection-id</option> (string)</term>
                     <listitem><para>The collection ID of the repository,
@@ -71,7 +87,27 @@
                     must match the existing configured collection ID for that
                     repository.</para></listitem>
                 </varlistentry>
-                -->
+                <varlistentry>
+                    <term><option>extension-tag</option> (string)</term>
+                    <listitem><para>If building an extension, the tag for the extension
+                    point to use. Since flatpak 0.11.4 a runtime may define multiple
+                    locations for the same extension point with the intention that
+                    different branches for the extension are mounted at each location. When
+                    building an extension it is necessary to know what extension point to
+                    install the extension to. This option resolves any ambiguity
+                    in which extension point to choose. If not specified, the default choice
+                    is to install into either the only location for the extension point or
+                    into the location for the untagged extension point. If there are multiple
+                    locations for the same extension point defined with different tags
+                    then an error will occur.</para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>token-type</option> (integer)</term>
+                    <listitem><para>The type of token needed to install
+                    this commit. Setting this to a value greater than 0
+                    implies that authentication will be needed to
+                    install the flatpak.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>runtime</option> (string)</term>
                     <listitem><para>The name of the runtime that the application uses.</para></listitem>
@@ -94,7 +130,16 @@
                 </varlistentry>
                 <varlistentry>
                     <term><option>command</option> (string)</term>
-                    <listitem><para>The filename or path to the main binary of the application. Note that this is really just a single file, not a commandline. If you want to pass arguments, install a shell script wrapper and use that as the command.</para></listitem>
+                    <listitem><para>
+                        The filename or path to the main binary of the application. Note that this
+                        is really just a single file, not a commandline. If you want to pass arguments,
+                        install a shell script wrapper and use that as the command.
+                    </para><para>
+                        Also note that the command is used when the application is run via
+                        <command>flatpak run</command>, and does not affect what gets executed when
+                        the application is run in other ways, e.g. via the desktop file or D-Bus
+                        activation.
+                    </para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>build-runtime</option> (boolean)</term>
@@ -155,7 +200,7 @@
                     finishing the build.</para></listitem>
                 </varlistentry>
                 <varlistentry>
-                    <term><option>inherit-sd-extensions</option> (array of strings)</term>
+                    <term><option>inherit-sdk-extensions</option> (array of strings)</term>
                     <listitem><para>Inherit these extra extensions points from the base application or sdk when
                     finishing the build, but do not inherit them into the platform.</para></listitem>
                 </varlistentry>
@@ -169,15 +214,19 @@
                     <listitem><para>Object specifying the build environment. See below for details.</para></listitem>
                 </varlistentry>
                 <varlistentry>
-                    <term><option>modules</option> (array of objects or string)</term>
+                    <term><option>modules</option> (array of objects or strings)</term>
                     <listitem><para>An array of objects specifying the modules to be built in order.
-                    String members in the array are interpreted as the name of a separate json file that contains a module.
+                    String members in the array are interpreted as the name of a separate json or yaml file that contains a module.
                     See below for details.</para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>add-extensions</option> (objects)</term>
                     <listitem><para>This is a dictionary of extension objects. The key is the name of the extension. See below for details.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>add-build-extensions</option> (objects)</term>
+                    <listitem><para>This is a dictionary of extension objects similar to add-extensions. The main difference is that the extensions are added early and are available for use during the build.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>cleanup</option> (array of strings)</term>
                     <listitem><para>An array of file patterns that should be removed at the end.
@@ -254,23 +303,44 @@
                     <term><option>cflags</option> (string)</term>
                     <listitem><para>This is set in the environment variable CFLAGS during the build. Multiple specifications of this (in e.g. per-arch area) are concatenated, separated by spaces.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>cflags-override</option> (boolean)</term>
+                    <listitem><para>If this is true, clear cflags from previous build options before adding it from these options.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>cppflags</option> (string)</term>
                     <listitem><para>This is set in the environment variable CPPFLAGS during the build. Multiple specifications of this (in e.g. per-arch area) are concatenated, separated by spaces.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>cppflags-override</option> (boolean)</term>
+                    <listitem><para>If this is true, clear cppflags from previous build options before adding it from these options.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>cxxflags</option> (string)</term>
                     <listitem><para>This is set in the environment variable CXXFLAGS during the build. Multiple specifications of this (in e.g. per-arch area) are concatenated, separated by spaces.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>cxxflags-override</option> (boolean)</term>
+                    <listitem><para>If this is true, clear cxxflags from previous build options before adding it from these options.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>ldflags</option> (string)</term>
                     <listitem><para>This is set in the environment variable LDFLAGS during the build. Multiple specifications of this (in e.g. per-arch area) are concatenated, separated by spaces.</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>ldflags-override</option> (boolean)</term>
+                    <listitem><para>If this is true, clear ldflags from previous build options before adding it from these options.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>prefix</option> (string)</term>
                     <listitem><para>The build prefix for the modules (defaults to <filename>/app</filename> for
                     applications and <filename>/usr</filename> for runtimes).</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>libdir</option> (string)</term>
+                    <listitem><para>The build libdir for the modules (defaults to <filename>/app/lib</filename> for
+                    applications and <filename>/usr/lib</filename> for runtimes).</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>append-path</option> (string)</term>
                     <listitem><para>This will get appended to PATH in the build environment (with an leading colon if
@@ -303,7 +373,7 @@
                 </varlistentry>
                 <varlistentry>
                     <term><option>env</option> (object)</term>
-                    <listitem><para>This is a dictionary defining environment variables to be set during the build. Elements in this override the properties that set the environment, like cflags and ldflags.</para></listitem>
+                    <listitem><para>This is a dictionary defining environment variables to be set during the build. Elements in this override the properties that set the environment, like cflags and ldflags. Keys with a null value unset the corresponding variable. </para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>build-args</option> (array of strings)</term>
@@ -356,22 +426,34 @@
             <variablelist>
                 <varlistentry>
                     <term><option>directory</option> (string)</term>
-                    <listitem><para>The directory where the extension is mounted.</para></listitem>
+                    <listitem><para>
+                      The directory where the extension is mounted.
+                      If the extension point is for an application, this path is relative to /app,
+                      otherwise it is relative to /usr.
+                    </para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>bundle</option> (boolean)</term>
                     <listitem><para>If this is true, then the data
                     created in the extension directory is omitted from
                     the result, and instead packaged in a separate
-                    extension..</para></listitem>
+                    extension.</para></listitem>
+                </varlistentry>
+                <varlistentry>
+                    <term><option>remove-after-build</option> (boolean)</term>
+                    <listitem><para>If this is true, the extension is
+                    removed during when finishing. This is only
+                    interesting for extensions in the
+                    add-build-extensions property.</para></listitem>
                 </varlistentry>
             </variablelist>
             <para>
-              Additionally the standard flatpak extension properies
+              Additionally the standard flatpak extension properties
               are supported, and put directly into the metadata file:
               autodelete, no-autodownload, subdirectories,
               add-ld-path, download-if, enable-if, merge-dirs,
               subdirectory-suffix, locale-subset, version, versions.
+              See the flatpak metadata documentation for more information on these.
             </para>
         </refsect2>
         <refsect2>
@@ -396,11 +478,11 @@
                     <listitem><para>If true, skip this module</para></listitem>
                 </varlistentry>
                 <varlistentry>
-                    <term><option>sources</option> (array of objects or string)</term>
+                    <term><option>sources</option> (array of objects or strings)</term>
                     <listitem><para>An array of objects defining
                     sources that will be downloaded and extracted in
                     order. String members in the array are interpreted
-                    as the name of a separate json file that contains
+                    as the name of a separate json or yaml file that contains
                     sources.  See below for details.</para></listitem>
                 </varlistentry>
                 <varlistentry>
@@ -445,7 +527,7 @@
                 </varlistentry>
                 <varlistentry>
                     <term><option>buildsystem</option> (string)</term>
-                    <listitem><para>Build system to use: autotools, cmake, cmake-ninja, meson, simple</para></listitem>
+                    <listitem><para>Build system to use: autotools, cmake, cmake-ninja, meson, simple, qmake</para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>builddir</option> (boolean)</term>
@@ -463,6 +545,7 @@
                     <term><option>build-commands</option> (array of strings)</term>
                     <listitem><para>An array of commands to run during build (between make and make install if those are used).
                     This is primarily useful when using the "simple" buildsystem.
+                    Each command is run in <literal>/bin/sh -c</literal>, so it can use standard POSIX shell syntax such as piping output.
                     </para></listitem>
                 </varlistentry>
                 <varlistentry>
@@ -507,13 +590,13 @@
                     <listitem><para>The target to build when running the tests. Defaults to "check" for make and "test" for ninja. Set to empty to disable.</para></listitem>
                 </varlistentry>
                 <varlistentry>
-                    <term><option>test-commands</option> (arrya of string)</term>
+                    <term><option>test-commands</option> (array of strings)</term>
                     <listitem><para>Array of commands to run during the tests.</para></listitem>
                 </varlistentry>
                 <varlistentry>
                     <term><option>modules</option> (array of objects or strings)</term>
                     <listitem><para>An array of objects specifying nested modules to be built before this one.
-                    String members in the array are interpreted as names of a separate json file that contains a module.</para></listitem>
+                    String members in the array are interpreted as names of a separate json or yaml file that contains a module.</para></listitem>
                 </varlistentry>
             </variablelist>
         </refsect2>
@@ -525,7 +608,7 @@
             </para>
             <para>
                 Additionally, the sources list can contain a plain string, which is interpreted as the name
-                of a separate json file that is read and inserted at this point. The json file can contain
+                of a separate json or yaml file that is read and inserted at this point. The file can contain
                 a single source, or an array of sources.
             </para>
             <refsect3>
@@ -560,6 +643,21 @@
                         <term><option>url</option> (string)</term>
                         <listitem><para>The URL of a remote archive that will be downloaded. This overrides path if both are specified.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>mirror-urls</option> (array of strings)</term>
+                        <listitem><para>A list of alternative urls that are used if the main url fails.</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>git-init</option> (boolean)</term>
+                        <listitem><para>Whether to initialise the repository as a git repository.</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>archive-type</option> (string)</term>
+                        <listitem><para>
+                            The type of archive if it cannot be guessed from the path. Possible values are "rpm", "tar",
+                            "tar-gzip", "tar-compress", "tar-bzip2", "tar-lzip", "tar-lzma", "tar-lzop", "tar-xz", "zip" and "7z".
+                        </para></listitem>
+                    </varlistentry>
                     <varlistentry>
                         <term><option>md5</option> (string)</term>
                         <listitem><para>The md5 checksum of the file, verified after download</para>
@@ -584,6 +682,10 @@
                         <term><option>strip-components</option> (integer)</term>
                         <listitem><para>The number of initial pathname components to strip during extraction. Defaults to 1.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>dest-filename</option> (string)</term>
+                        <listitem><para>Filename to for the downloaded file, defaults to the basename of url.</para></listitem>
+                    </varlistentry>
                 </variablelist>
             </refsect3>
             <refsect3>
@@ -599,7 +701,7 @@
                     </varlistentry>
                     <varlistentry>
                         <term><option>url</option> (string)</term>
-                        <listitem><para>URL of the git repository. This overrides path if both are specified.</para></listitem>
+                        <listitem><para>URL of the git repository. This overrides path if both are specified. When using git via SSH, the correct syntax is ssh://user@domain/path/to/repo.git.</para></listitem>
                     </varlistentry>
                     <varlistentry>
                         <term><option>branch</option> (string)</term>
@@ -622,6 +724,10 @@
                         <term><option>disable-shallow-clone</option> (boolean)</term>
                         <listitem><para>Don't optimize by making a shallow clone when downloading the git repo.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>disable-submodules</option> (boolean)</term>
+                        <listitem><para>Don't checkout the git submodules when cloning the repository.</para></listitem>
+                    </varlistentry>
                 </variablelist>
             </refsect3>
             <refsect3>
@@ -641,6 +747,40 @@
                     </varlistentry>
                 </variablelist>
             </refsect3>
+            <refsect3>
+                <title>Svn sources</title>
+                <variablelist>
+                    <varlistentry>
+                        <term><option>type</option></term>
+                        <listitem><para>"svn"</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>url</option> (string)</term>
+                        <listitem><para>URL of the svn repository, including branch/tag part</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>revision</option> (string)</term>
+                        <listitem><para>A specific revision number to use</para></listitem>
+                    </varlistentry>
+                </variablelist>
+            </refsect3>
+            <refsect3>
+                <title>Directory sources</title>
+                <variablelist>
+                    <varlistentry>
+                        <term><option>type</option></term>
+                        <listitem><para>"dir"</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>path</option> (string)</term>
+                        <listitem><para>The path of a local directory whose content will be copied into the source dir. Note that directory sources don't currently support caching, so they will be rebuilt each time.</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                        <term><option>skip</option> (array of strings)</term>
+                        <listitem><para>Source files to ignore in the directory.</para></listitem>
+                    </varlistentry>
+                </variablelist>
+            </refsect3>
             <refsect3>
                 <title>File sources</title>
                 <variablelist>
@@ -656,6 +796,10 @@
                         <term><option>url</option> (string)</term>
                         <listitem><para>The URL of a remote file that will be downloaded and copied into the source dir. This overrides path if both are specified.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>mirror-urls</option> (array of strings)</term>
+                        <listitem><para>A list of alternative urls that are used if the main url fails.</para></listitem>
+                    </varlistentry>
                     <varlistentry>
                         <term><option>md5</option> (string)</term>
                         <listitem><para>The md5 checksum of the file, verified after download. This is optional for local files.</para>
@@ -698,7 +842,7 @@
                     </varlistentry>
                     <varlistentry>
                         <term><option>dest-filename</option> (string)</term>
-                        <listitem><para>Filename to use inside the source dir, default to the basename of path.</para></listitem>
+                        <listitem><para>Filename to use inside the source dir, default to autogen.sh.</para></listitem>
                     </varlistentry>
                 </variablelist>
             </refsect3>
@@ -729,6 +873,10 @@
                         <term><option>path</option> (string)</term>
                         <listitem><para>The path of a patch file that will be applied in the source dir</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>paths</option> (array of strings)</term>
+                        <listitem><para>An list of paths to a patch files that will be applied in the source dir, in order</para></listitem>
+                    </varlistentry>
                     <varlistentry>
                         <term><option>strip-components</option> (integer)</term>
                         <listitem><para>The value of the -p argument to patch, defaults to 1.</para></listitem>
@@ -737,6 +885,10 @@
                         <term><option>use-git</option> (boolean)</term>
                         <listitem><para>Whether to use "git apply" rather than "patch" to apply the patch, required when the patch file contains binary diffs.</para></listitem>
                     </varlistentry>
+                    <varlistentry>
+                        <term><option>use-git-am</option> (boolean)</term>
+                        <listitem><para>Whether to use "git am" rather than "patch" to apply the patch, required when the patch file contains binary diffs. You cannot use this at the same time as <option>use-git</option>.</para></listitem>
+                    </varlistentry>
                     <varlistentry>
                         <term><option>options</option> (array of strings)</term>
                         <listitem><para>Extra options to pass to the patch command.</para></listitem>
@@ -763,7 +915,7 @@
                         <listitem><para>The sha256 of the extra data.</para></listitem>
                     </varlistentry>
                     <varlistentry>
-                        <term><option>size</option> (string)</term>
+                        <term><option>size</option> (number)</term>
                         <listitem><para>The size of the extra data.</para></listitem>
                     </varlistentry>
                     <varlistentry>
@@ -775,6 +927,109 @@
         </refsect2>
     </refsect1>
 
+    <refsect1>
+      <title>Build environment</title>
+
+      <para>
+        When building the application each command is run in a separate sandbox with access
+        to only the things required for it. This section describes the details of the sandbox.
+        Any options here can be overridden globally or per-module with the <option>build-args</option>
+        option (although such manifest will not work if you start flatpak-builder with <option>--sandbox</option>).
+      </para>
+
+      <refsect2>
+        <title>Filesystem</title>
+        <para>
+          Each module is built in its own build directory, stored in a sub directory called
+          <filename>build/$modulename-$count</filename> in the state dir (which is typically <filename>.flatpak-builder/</filename>).
+          Additionally there is a symlink <filename>build/$modulename</filename> to the latest version.
+          In order to generate reproducible builds this directory is also mounted as <filename>/run/build/$modulename</filename>
+          in the sandbox (or <filename>/run/build-runtime/$modulename</filename> when building runtimes).
+          This is used as current working directory for all build ops.
+        </para>
+        <para>
+          The destination directory for installation is accessible for writing at the place it will seen at runtime.
+          In the case of a regular application this will be /app. If building a runtime it will instead be /usr, and
+          when building an extension it will be at the extensionpoint directory somewhere below /app (for app extension)
+          or /usr (for runtime extensions).
+        </para>
+        <para>
+          Additionally the there will be (as needed, depending on what is building) read-only mounts of the sdk at /usr,
+          sdk extensions below that, and the application at /app. No other filesystem access is available.
+        </para>
+      </refsect2>
+
+      <refsect2>
+        <title>Environment</title>
+        <para>
+          The environment can be modified in several ways in the manifest, but the default values are:
+        </para>
+        <variablelist>
+          <varlistentry>
+            <term>FLATPAK_ID</term>
+            <listitem><para>The id of the application currently building.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>FLATPAK_ARCH</term>
+            <listitem><para>The architecture currently building.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>FLATPAK_DEST</term>
+            <listitem><para>The path to where the current build should install into. This is <filename>/app</filename> for application builds.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>FLATPAK_BUILDER_N_JOBS</term>
+            <listitem><para>The number of jobs that flatpak-builder would normally use for make -j. Defaults to ncpus unless the module disabled parallel make.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>FLATPAK_BUILDER_BUILDDIR</term>
+            <listitem><para>The path to the build directory of the module currently building. This is normally <filename>/run/build/$MODULE</filename>.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>PATH</term>
+            <listitem><para><filename>/app/bin:/usr/bin</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>LD_LIBRARY_PATH</term>
+            <listitem><para><filename>/app/lib</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>PKG_CONFIG_PATH</term>
+            <listitem><para><filename>/app/lib/pkgconfig:/app/share/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>ACLOCAL_PATH</term>
+            <listitem><para><filename>/app/share/aclocal</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>C_INCLUDE_PATH</term>
+            <listitem><para><filename>/app/include</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>CPLUS_INCLUDE_PATH</term>
+            <listitem><para><filename>/app/include</filename></para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>LDFLAGS</term>
+            <listitem><para>-L/app/lib </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>LC_ALL</term>
+            <listitem><para>en_US.utf8</para></listitem>
+          </varlistentry>
+        </variablelist>
+      </refsect2>
+
+      <refsect2>
+        <title>Permissions</title>
+        <para>
+          Builds have the --allow=devel and --allow=multiarch permissions that regular flatpak runs don't have by default. This allows
+          limits the syscall filtering that is normally done so development tools like debuggers work. Otherwise the build sandbox
+          is very limited, for example there is no network access.
+        </para>
+      </refsect2>
+    </refsect1>
+
     <refsect1>
         <title>Examples</title>
 
@@ -833,7 +1088,7 @@
             "sources": [
                 {
                     "type": "git",
-                    "url": "git://git.gnome.org/babl"
+                    "url": "https://gitlab.gnome.org/GNOME/babl.git"
                 }
             ]
         },

libglnx

@@ -1 +1 @@
-Subproject commit 0c82203cd459a35cc3f471e3205355e9fb79160f
+Subproject commit 470af8763ff7b99bec950a6ae0a957c1dcfc8edd

src/Makefile.am.inc

@@ -22,6 +22,8 @@ flatpak_builder_SOURCES = \
 	src/builder-source-git.h \
 	src/builder-source-bzr.c \
 	src/builder-source-bzr.h \
+	src/builder-source-svn.c \
+	src/builder-source-svn.h \
 	src/builder-source-file.c \
 	src/builder-source-file.h \
 	src/builder-source-script.c \
@@ -32,6 +34,8 @@ flatpak_builder_SOURCES = \
 	src/builder-source-extra-data.h \
 	src/builder-source-patch.c \
 	src/builder-source-patch.h \
+	src/builder-source-dir.c \
+	src/builder-source-dir.h \
 	src/builder-context.c \
 	src/builder-context.h \
 	src/builder-cache.c \
@@ -42,7 +46,9 @@ flatpak_builder_SOURCES = \
 	src/builder-flatpak-utils.h \
 	src/builder-git.c \
 	src/builder-git.h \
+	src/builder-sdk-config.c \
+	src/builder-sdk-config.h \
 	$(NULL)
 
-flatpak_builder_LDADD = $(AM_LDADD) $(BASE_LIBS) $(LIBELF_LIBS) libglnx.la
-flatpak_builder_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS)
+flatpak_builder_LDADD = $(AM_LDADD) $(BASE_LIBS) $(LIBELF_LIBS) $(YAML_LIBS) libglnx.la
+flatpak_builder_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS) $(YAML_CFLAGS)

src/builder-cache.c

@@ -210,13 +210,10 @@ builder_cache_get_checksum (BuilderCache *self)
   return self->checksum;
 }
 
-static char *
-get_ref (BuilderCache *self, const char *stage)
+static void
+append_escaped_stage (GString *s,
+                      const char *stage)
 {
-  GString *s = g_string_new (self->branch);
-
-  g_string_append_c (s, '/');
-
   while (*stage)
     {
       char c = *stage++;
@@ -228,6 +225,16 @@ get_ref (BuilderCache *self, const char *stage)
       else
         g_string_append_printf (s, "%x", c);
     }
+}
+
+static char *
+get_ref (BuilderCache *self, const char *stage)
+{
+  GString *s = g_string_new (self->branch);
+
+  g_string_append_c (s, '/');
+
+  append_escaped_stage (s, stage);
 
   return g_string_free (s, FALSE);
 }
@@ -376,11 +383,13 @@ builder_cache_lookup (BuilderCache *self,
 {
   g_autofree char *commit = NULL;
   g_autofree char *ref = NULL;
+  g_autoptr(GString) s = g_string_new ("");
 
   g_free (self->stage);
   self->stage = g_strdup (stage);
 
-  g_hash_table_remove (self->unused_stages, stage);
+  append_escaped_stage (s, stage);
+  g_hash_table_remove (self->unused_stages, s->str);
 
   g_free (self->current_checksum);
   self->current_checksum = g_strdup (g_checksum_get_string (self->checksum));
@@ -579,6 +588,10 @@ builder_cache_commit (BuilderCache *self,
   g_autoptr(GPtrArray) removals = NULL;
   g_autoptr(GVariantDict) metadata_dict = NULL;
   g_autoptr(GVariant) metadata = NULL;
+  g_autoptr(GVariant) changesv = NULL;
+  g_autoptr(GVariant) removalsv = NULL;
+  g_autoptr(GVariant) changesvz = NULL;
+  g_autoptr(GVariant) removalsvz = NULL;
 
   g_print ("Committing stage %s to cache\n", self->stage);
 
@@ -606,11 +619,17 @@ builder_cache_commit (BuilderCache *self,
     goto out;
 
   changes = builder_cache_get_changes_to (self, root, &removals, NULL);
+
   metadata_dict = g_variant_dict_new (NULL);
-  g_variant_dict_insert_value (metadata_dict, "changes",
-                               g_variant_new_strv ((const gchar * const  *) changes->pdata, changes->len));
-  g_variant_dict_insert_value (metadata_dict, "removals",
-                               g_variant_new_strv ((const gchar * const  *) removals->pdata, removals->len));
+
+  changesv = g_variant_ref_sink (g_variant_new_strv ((const gchar * const  *) changes->pdata, changes->len));
+  changesvz = flatpak_variant_compress (changesv);
+  g_variant_dict_insert_value (metadata_dict, "changesz", changesvz);
+
+  removalsv = g_variant_ref_sink (g_variant_new_strv ((const gchar * const  *) removals->pdata, removals->len));
+  removalsvz = flatpak_variant_compress (removalsv);
+  g_variant_dict_insert_value (metadata_dict, "removalsz", removalsvz);
+
   metadata = g_variant_ref_sink (g_variant_dict_end (metadata_dict));
 
   current = self->current_checksum;
@@ -1153,6 +1172,7 @@ builder_cache_get_changes (BuilderCache *self,
   g_autoptr(GVariant) variant = NULL;
   g_autoptr(GVariant) commit_metadata = NULL;
   g_autofree char *parent_commit = NULL;
+  g_autoptr(GVariant) changesz_v = NULL;
   g_autoptr(GVariant) changes_v = NULL;
 
   if (!ostree_repo_read_commit (self->repo, self->last_parent, &current_root, NULL, NULL, error))
@@ -1163,7 +1183,12 @@ builder_cache_get_changes (BuilderCache *self,
     return NULL;
 
   commit_metadata = g_variant_get_child_value (variant, 0);
-  changes_v = g_variant_lookup_value (commit_metadata, "changes", G_VARIANT_TYPE ("as"));
+  changesz_v = g_variant_lookup_value (commit_metadata, "changesz", G_VARIANT_TYPE_BYTESTRING);
+
+  if (changesz_v)
+    changes_v = flatpak_variant_uncompress (changesz_v, G_VARIANT_TYPE ("as"));
+  else
+    changes_v = g_variant_lookup_value (commit_metadata, "changes", G_VARIANT_TYPE ("as"));
 
   if (changes_v)
     {
@@ -1210,6 +1235,7 @@ builder_cache_disable_lookups (BuilderCache *self)
 
 gboolean
 builder_gc (BuilderCache *self,
+            gboolean      prune_unused_stages,
             GError      **error)
 {
   gint objects_total;
@@ -1218,20 +1244,23 @@ builder_gc (BuilderCache *self,
   GHashTableIter iter;
   gpointer key, value;
 
-  g_hash_table_iter_init (&iter, self->unused_stages);
-  while (g_hash_table_iter_next (&iter, &key, &value))
+  if (prune_unused_stages)
     {
-      const char *unused_stage = (const char *) key;
-      g_autofree char *unused_ref = get_ref (self, unused_stage);
+      g_hash_table_iter_init (&iter, self->unused_stages);
+      while (g_hash_table_iter_next (&iter, &key, &value))
+        {
+          const char *unused_stage = (const char *) key;
+          g_autofree char *unused_ref = get_ref (self, unused_stage);
 
-      g_debug ("Removing unused ref %s", unused_ref);
+          g_debug ("Removing unused ref %s", unused_ref);
 
-      if (!ostree_repo_set_ref_immediate (self->repo,
-                                          NULL,
-                                          unused_ref,
-                                          NULL,
-                                          NULL, error))
-        return FALSE;
+          if (!ostree_repo_set_ref_immediate (self->repo,
+                                              NULL,
+                                              unused_ref,
+                                              NULL,
+                                              NULL, error))
+            return FALSE;
+        }
     }
 
   g_print ("Pruning cache\n");
@@ -1336,6 +1365,13 @@ builder_cache_checksum_uint32 (BuilderCache *self,
   g_checksum_update (self->checksum, v, 4);
 }
 
+void
+builder_cache_checksum_random (BuilderCache *self)
+{
+  builder_cache_checksum_uint32 (self, g_random_int ());
+  builder_cache_checksum_uint32 (self, g_random_int ());
+}
+
 void
 builder_cache_checksum_uint64 (BuilderCache *self,
                                guint64       val)

src/builder-cache.h

@@ -59,6 +59,7 @@ GPtrArray   *builder_cache_get_changes (BuilderCache *self,
 GPtrArray   *builder_cache_get_all_changes (BuilderCache *self,
                                             GError      **error);
 gboolean      builder_gc (BuilderCache *self,
+                          gboolean      prune_unused_stages,
                           GError      **error);
 
 void builder_cache_checksum_str (BuilderCache *self,
@@ -80,6 +81,7 @@ void builder_cache_checksum_uint64 (BuilderCache *self,
 void builder_cache_checksum_data (BuilderCache *self,
                                   guint8       *data,
                                   gsize         len);
+void builder_cache_checksum_random (BuilderCache *self);
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderCache, g_object_unref)
 

src/builder-context.c

@@ -46,8 +46,11 @@ struct BuilderContext
   GFile          *base_dir; /* directory with json manifest, origin for source files */
   char           *state_subdir;
   SoupSession    *soup_session;
+  CURL           *curl_session;
   char           *arch;
+  char           *default_branch;
   char           *stop_at;
+  gint64          source_date_epoch;
 
   GFile          *download_dir;
   GPtrArray      *sources_dirs;
@@ -78,6 +81,8 @@ struct BuilderContext
   gboolean        have_rofiles;
   gboolean        run_tests;
   gboolean        no_shallow_clone;
+
+  BuilderSdkConfig *sdk_config;
 };
 
 typedef struct
@@ -113,7 +118,9 @@ builder_context_finalize (GObject *object)
   g_clear_object (&self->base_dir);
   g_clear_object (&self->soup_session);
   g_clear_object (&self->options);
+  g_clear_object (&self->sdk_config);
   g_free (self->arch);
+  g_free (self->default_branch);
   g_free (self->state_subdir);
   g_free (self->stop_at);
   g_strfreev (self->cleanup);
@@ -123,6 +130,10 @@ builder_context_finalize (GObject *object)
   g_clear_pointer (&self->sources_dirs, g_ptr_array_unref);
   g_clear_pointer (&self->sources_urls, g_ptr_array_unref);
 
+  curl_easy_cleanup (self->curl_session);
+  self->curl_session = NULL;
+  curl_global_cleanup ();
+
   G_OBJECT_CLASS (builder_context_parent_class)->finalize (object);
 }
 
@@ -191,7 +202,18 @@ builder_context_constructed (GObject *object)
   self->build_dir = g_file_get_child (self->state_dir, "build");
   self->cache_dir = g_file_get_child (self->state_dir, "cache");
   self->checksums_dir = g_file_get_child (self->state_dir, "checksums");
-  self->ccache_dir = g_file_get_child (self->state_dir, "ccache");
+
+  // Check, if CCACHE_DIR is set in environment and use it, instead of subdir of state_dir
+  const char * env_ccache_dir = g_getenv ("CCACHE_DIR");
+  if (env_ccache_dir && g_path_is_absolute(env_ccache_dir))
+    {
+      g_debug ("Using CCACHE_DIR '%s'", env_ccache_dir);
+      self->ccache_dir = g_file_new_for_path (env_ccache_dir);
+    }
+    else
+    {
+      self->ccache_dir = g_file_get_child(self->state_dir, "ccache");
+    }
 }
 
 static void
@@ -354,6 +376,7 @@ builder_context_set_sources_urls (BuilderContext *self,
 gboolean
 builder_context_download_uri (BuilderContext *self,
                               const char     *url,
+                              const char    **mirrors,
                               GFile          *dest,
                               const char     *checksums[BUILDER_CHECKSUMS_LEN],
                               GChecksumType   checksums_type[BUILDER_CHECKSUMS_LEN],
@@ -361,6 +384,7 @@ builder_context_download_uri (BuilderContext *self,
 {
   int i;
   g_autoptr(SoupURI) original_uri = soup_uri_new (url);
+  g_autoptr(GError) first_error = NULL;
 
   if (original_uri == NULL)
     return flatpak_fail (error, _("Could not parse URI “%s”"), url);
@@ -383,11 +407,11 @@ builder_context_download_uri (BuilderContext *self,
           if (builder_download_uri (mirror_uri,
                                     dest,
                                     checksums, checksums_type,
-                                    builder_context_get_soup_session (self),
+                                    builder_context_get_curl_session (self),
                                     &my_error))
             return TRUE;
 
-          if (!g_error_matches (my_error, SOUP_HTTP_ERROR, SOUP_STATUS_NOT_FOUND))
+          if (!g_error_matches (my_error, BUILDER_CURL_ERROR, CURLE_REMOTE_FILE_NOT_FOUND))
             g_warning ("Error downloading from mirror: %s\n", my_error->message);
         }
     }
@@ -395,9 +419,41 @@ builder_context_download_uri (BuilderContext *self,
   if (!builder_download_uri (original_uri,
                              dest,
                              checksums, checksums_type,
-                             builder_context_get_soup_session (self),
-                             error))
-    return FALSE;
+                             builder_context_get_curl_session (self),
+                             &first_error))
+    {
+      gboolean mirror_ok = FALSE;
+
+      if (mirrors != NULL && mirrors[0] != NULL)
+        {
+          g_print ("Error downloading, trying mirrors\n");
+          for (i = 0; mirrors[i] != NULL; i++)
+            {
+              g_autoptr(GError) mirror_error = NULL;
+              g_autoptr(SoupURI) mirror_uri = soup_uri_new (mirrors[i]);
+              g_print ("Trying mirror %s\n", mirrors[i]);
+              if (!builder_download_uri (mirror_uri,
+                                         dest,
+                                         checksums, checksums_type,
+                                         builder_context_get_curl_session (self),
+                                         &mirror_error))
+                {
+                  g_print ("Error downloading mirror: %s\n", mirror_error->message);
+                }
+              else
+                {
+                  mirror_ok = TRUE;
+                  break;
+                }
+            }
+        }
+
+      if (!mirror_ok)
+        {
+          g_propagate_error (error, g_steal_pointer (&first_error));
+          return FALSE;
+        }
+    }
 
   return TRUE;
 }
@@ -418,7 +474,8 @@ char *
 builder_context_get_checksum_for (BuilderContext *self,
                                   const char *name)
 {
-  g_autoptr(GFile) checksum_file = g_file_get_child (self->checksums_dir, name);
+  g_autofree char *checksum_name = g_strdup_printf ("%s-%s", builder_context_get_arch (self), name);
+  g_autoptr(GFile) checksum_file = g_file_get_child (self->checksums_dir, checksum_name);
   g_autofree gchar *checksum = NULL;
 
   if (!g_file_get_contents (flatpak_file_get_path_cached (checksum_file), &checksum, NULL, NULL))
@@ -433,7 +490,8 @@ builder_context_set_checksum_for (BuilderContext  *self,
                                   const char      *checksum,
                                   GError         **error)
 {
-  g_autoptr(GFile) checksum_file = g_file_get_child (self->checksums_dir, name);
+  g_autofree char *checksum_name = g_strdup_printf ("%s-%s", builder_context_get_arch (self), name);
+  g_autoptr(GFile) checksum_file = g_file_get_child (self->checksums_dir, checksum_name);
 
   if (!flatpak_mkdir_p (self->checksums_dir,
                         NULL, error))
@@ -495,6 +553,15 @@ builder_context_get_soup_session (BuilderContext *self)
   return self->soup_session;
 }
 
+CURL *
+builder_context_get_curl_session (BuilderContext *self)
+{
+  if (self->curl_session == NULL)
+    self->curl_session = flatpak_create_curl_session ("flatpak-builder " PACKAGE_VERSION);
+
+  return self->curl_session;
+}
+
 const char *
 builder_context_get_arch (BuilderContext *self)
 {
@@ -512,6 +579,27 @@ builder_context_set_arch (BuilderContext *self,
   self->arch = g_strdup (arch);
 }
 
+const char *
+builder_context_get_default_branch (BuilderContext *self)
+{
+  return (const char *) self->default_branch;
+}
+
+void
+builder_context_set_default_branch (BuilderContext *self,
+                                    const char     *default_branch)
+{
+  g_free (self->default_branch);
+  self->default_branch = g_strdup (default_branch);
+}
+
+void
+builder_context_set_source_date_epoch (BuilderContext *self,
+                                       gint64 source_date_epoch)
+{
+  self->source_date_epoch = source_date_epoch;
+}
+
 const char *
 builder_context_get_stop_at (BuilderContext *self)
 {
@@ -678,7 +766,7 @@ static char *rofiles_unmount_path = NULL;
 static void
 rofiles_umount_handler (int signum)
 {
-  char *argv[] = { "fusermount", "-u", NULL,
+  char *argv[] = { "fusermount", "-uz", NULL,
                      NULL };
 
   argv[2] = rofiles_unmount_path;
@@ -952,8 +1040,21 @@ builder_context_set_enable_ccache (BuilderContext *self,
 }
 
 char **
-builder_context_extend_env (BuilderContext *self,
-                            char          **envp)
+builder_context_extend_env_pre (BuilderContext *self,
+                                char          **envp)
+{
+  if (self->source_date_epoch != 0)
+    {
+      g_autofree char *s_d_e = g_strdup_printf ("%" G_GUINT64_FORMAT, self->source_date_epoch);
+      envp = g_environ_setenv (envp, "SOURCE_DATE_EPOCH", s_d_e, FALSE);
+    }
+
+  return envp;
+}
+
+char **
+builder_context_extend_env_post (BuilderContext *self,
+                                 char          **envp)
 {
   g_autofree char *path = NULL;
   const char *ccache_dir = NULL;
@@ -980,6 +1081,34 @@ builder_context_extend_env (BuilderContext *self,
   return envp;
 }
 
+gboolean
+builder_context_load_sdk_config (BuilderContext   *self,
+                                 const char       *sdk_path,
+                                 GError          **error)
+{
+  g_autoptr(GFile) root = g_file_new_for_path (sdk_path);
+  g_autoptr(GFile) config_file = g_file_resolve_relative_path (root, "files/etc/flatpak-builder/defaults.json");
+  g_autoptr(GError) local_error = NULL;
+  g_autoptr(BuilderSdkConfig) sdk_config = NULL;
+
+  sdk_config = builder_sdk_config_from_file (config_file, &local_error);
+  if (sdk_config == NULL &&
+      !g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+    {
+      g_propagate_error (error, g_steal_pointer (&local_error));
+      return FALSE;
+    }
+
+  g_set_object (&self->sdk_config, sdk_config);
+  return TRUE;
+}
+
+BuilderSdkConfig *
+builder_context_get_sdk_config (BuilderContext *self)
+{
+  return self->sdk_config;
+}
+
 BuilderContext *
 builder_context_new (GFile *run_dir,
                      GFile *app_dir,

src/builder-context.h

@@ -23,8 +23,10 @@
 
 #include <gio/gio.h>
 #include <libsoup/soup.h>
+#include <curl/curl.h>
 #include "builder-options.h"
 #include "builder-utils.h"
+#include "builder-sdk-config.h"
 
 G_BEGIN_DECLS
 
@@ -62,14 +64,21 @@ void            builder_context_set_sources_urls (BuilderContext *self,
                                                   GPtrArray      *sources_urls);
 gboolean        builder_context_download_uri (BuilderContext *self,
                                               const char     *url,
+                                              const char    **mirrors,
                                               GFile          *dest,
                                               const char     *checksums[BUILDER_CHECKSUMS_LEN],
                                               GChecksumType   checksums_type[BUILDER_CHECKSUMS_LEN],
                                               GError        **error);
 SoupSession *   builder_context_get_soup_session (BuilderContext *self);
+CURL *          builder_context_get_curl_session (BuilderContext *self);
 const char *    builder_context_get_arch (BuilderContext *self);
 void            builder_context_set_arch (BuilderContext *self,
                                           const char     *arch);
+const char *    builder_context_get_default_branch (BuilderContext *self);
+void            builder_context_set_default_branch (BuilderContext *self,
+                                                    const char     *default_branch);
+void            builder_context_set_source_date_epoch (BuilderContext *self,
+                                                       gint64 source_date_epoch);
 const char *    builder_context_get_stop_at (BuilderContext *self);
 void            builder_context_set_stop_at (BuilderContext *self,
                                              const char     *module);
@@ -136,8 +145,16 @@ void            builder_context_set_run_tests (BuilderContext *self,
 void            builder_context_set_no_shallow_clone (BuilderContext *self,
                                                       gboolean        no_shallow_clone);
 gboolean        builder_context_get_no_shallow_clone (BuilderContext *self);
-char **         builder_context_extend_env (BuilderContext *self,
-                                            char          **envp);
+char **         builder_context_extend_env_pre (BuilderContext *self,
+                                                 char          **envp);
+char **         builder_context_extend_env_post (BuilderContext *self,
+                                                 char          **envp);
+
+gboolean        builder_context_load_sdk_config (BuilderContext       *self,
+                                                 const char           *sdk_path,
+                                                 GError              **error);
+
+BuilderSdkConfig * builder_context_get_sdk_config (BuilderContext *self);
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderContext, g_object_unref)
 

src/builder-extension.c

@@ -53,6 +53,7 @@ struct BuilderExtension
   char           *subdirectory_suffix;
   char           *version;
   char           *versions;
+  gboolean        remove_after_build;
 };
 
 typedef struct
@@ -77,6 +78,7 @@ enum {
   PROP_SUBDIRECTORY_SUFFIX,
   PROP_VERSION,
   PROP_VERSIONS,
+  PROP_REMOVE_AFTER_BUILD,
   LAST_PROP
 };
 
@@ -116,6 +118,10 @@ builder_extension_get_property (GObject    *object,
       g_value_set_boolean (value, self->bundle);
       break;
 
+    case PROP_REMOVE_AFTER_BUILD:
+      g_value_set_boolean (value, self->remove_after_build);
+      break;
+
     case PROP_AUTODELETE:
       g_value_set_boolean (value, self->autodelete);
       break;
@@ -184,6 +190,10 @@ builder_extension_set_property (GObject      *object,
       self->bundle = g_value_get_boolean (value);
       break;
 
+    case PROP_REMOVE_AFTER_BUILD:
+      self->remove_after_build = g_value_get_boolean (value);
+      break;
+
     case PROP_AUTODELETE:
       self->autodelete = g_value_get_boolean (value);
       break;
@@ -263,6 +273,13 @@ builder_extension_class_init (BuilderExtensionClass *klass)
                                                          "",
                                                          FALSE,
                                                          G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_REMOVE_AFTER_BUILD,
+                                   g_param_spec_boolean ("remove-after-build",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_AUTODELETE,
                                    g_param_spec_boolean ("autodelete",
@@ -361,6 +378,11 @@ builder_extension_get_name (BuilderExtension *self)
   return self->name;
 }
 
+const char *
+builder_extension_get_version (BuilderExtension *self)
+{
+  return self->version;
+}
 
 gboolean
 builder_extension_is_bundled (BuilderExtension *self)
@@ -396,6 +418,15 @@ add_argb (BuilderExtension  *self,
     add_arg (self, args, key, "true");
 }
 
+void
+builder_extension_add_remove_args (BuilderExtension  *self,
+                                   GPtrArray *args)
+{
+  if (self->remove_after_build)
+    g_ptr_array_add (args,
+                     g_strdup_printf ("--remove-extension=%s", self->name));
+}
+
 void
 builder_extension_add_finish_args (BuilderExtension  *self,
                                    GPtrArray *args)
@@ -440,4 +471,5 @@ builder_extension_checksum (BuilderExtension  *self,
   builder_cache_checksum_str (cache, self->subdirectory_suffix);
   builder_cache_checksum_str (cache, self->version);
   builder_cache_checksum_str (cache, self->versions);
+  builder_cache_checksum_compat_boolean (cache, self->remove_after_build);
 }

src/builder-extension.h

@@ -42,11 +42,14 @@ GType builder_extension_get_type (void);
 void   builder_extension_set_name (BuilderExtension *self,
                                    const char *name);
 const char * builder_extension_get_name (BuilderExtension *self);
+const char * builder_extension_get_version (BuilderExtension *self);
 
 gboolean builder_extension_is_bundled (BuilderExtension *self);
 const char * builder_extension_get_directory (BuilderExtension *self);
 
-void    builder_extension_add_finish_args (BuilderExtension  *self,
+void     builder_extension_add_remove_args (BuilderExtension  *self,
+                                            GPtrArray *args);
+void     builder_extension_add_finish_args (BuilderExtension  *self,
                                            GPtrArray *args);
 void     builder_extension_checksum (BuilderExtension  *self,
                                      BuilderCache   *cache,

src/builder-flatpak-utils.c

@@ -565,6 +565,8 @@ flatpak_spawnv (GFile                *dir,
 
   launcher = g_subprocess_launcher_new (0);
 
+  g_subprocess_launcher_setenv (launcher, "LANGUAGE", "C", TRUE);
+
   if (output)
     flags |= G_SUBPROCESS_FLAGS_STDOUT_PIPE;
 
@@ -693,6 +695,7 @@ gboolean
 flatpak_cp_a (GFile         *src,
               GFile         *dest,
               FlatpakCpFlags flags,
+              GPtrArray     *skip_files,
               GCancellable  *cancellable,
               GError       **error)
 {
@@ -763,14 +766,29 @@ flatpak_cp_a (GFile         *src,
     {
       const char *name = g_file_info_get_name (child_info);
       g_autoptr(GFile) src_child = g_file_get_child (src, name);
+      gboolean skip = FALSE;
+      int i;
+
+      for (i = 0; skip_files != NULL && i < skip_files->len; i++)
+        {
+          if (g_file_equal (src_child, g_ptr_array_index (skip_files, i)))
+            {
+              skip = TRUE;
+              break;
+            }
+        }
 
       if (dest_child)
         g_object_unref (dest_child);
       dest_child = g_file_get_child (dest, name);
 
-      if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
+      if (skip)
         {
-          if (!flatpak_cp_a (src_child, dest_child, flags,
+          /* skip src */
+        }
+      else if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
+        {
+          if (!flatpak_cp_a (src_child, dest_child, flags, skip_files,
                              cancellable, error))
             goto out;
         }
@@ -1055,137 +1073,6 @@ flatpak_allocate_tmpdir (int           tmpdir_dfd,
 }
 
 
-typedef struct {
-  GMainLoop *loop;
-  GError *error;
-  GOutputStream *out;
-  guint64 downloaded_bytes;
-  GString *content;
-  char buffer[16*1024];
-  FlatpakLoadUriProgress progress;
-  GCancellable *cancellable;
-  gpointer user_data;
-  guint64 last_progress_time;
-  char *etag;
-} LoadUriData;
-
-static void
-stream_closed (GObject *source, GAsyncResult *res, gpointer user_data)
-{
-  LoadUriData *data = user_data;
-  GInputStream *stream = G_INPUT_STREAM (source);
-  g_autoptr(GError) error = NULL;
-
-  if (!g_input_stream_close_finish (stream, res, &error))
-    g_warning ("Error closing http stream: %s", error->message);
-
-  g_main_loop_quit (data->loop);
-}
-
-static void
-load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data)
-{
-  LoadUriData *data = user_data;
-  GInputStream *stream = G_INPUT_STREAM (source);
-  gsize nread;
-
-  nread = g_input_stream_read_finish (stream, res, &data->error);
-  if (nread == -1 || nread == 0)
-    {
-      if (data->progress)
-        data->progress (data->downloaded_bytes, data->user_data);
-      g_input_stream_close_async (stream,
-                                  G_PRIORITY_DEFAULT, NULL,
-                                  stream_closed, data);
-      return;
-    }
-
-  if (data->out != NULL)
-    {
-      gsize n_written;
-
-      if (!g_output_stream_write_all (data->out, data->buffer, nread, &n_written,
-                                      NULL, &data->error))
-        {
-          data->downloaded_bytes += n_written;
-          g_input_stream_close_async (stream,
-                                      G_PRIORITY_DEFAULT, NULL,
-                                      stream_closed, data);
-          return;
-        }
-
-      data->downloaded_bytes += n_written;
-    }
-  else
-    {
-      data->downloaded_bytes += nread;
-      g_string_append_len (data->content, data->buffer, nread);
-    }
-
-  if (g_get_monotonic_time () - data->last_progress_time > 1 * G_USEC_PER_SEC)
-    {
-      if (data->progress)
-        data->progress (data->downloaded_bytes, data->user_data);
-      data->last_progress_time = g_get_monotonic_time ();
-    }
-
-  g_input_stream_read_async (stream, data->buffer, sizeof (data->buffer),
-                             G_PRIORITY_DEFAULT, data->cancellable,
-                             load_uri_read_cb, data);
-}
-
-static void
-load_uri_callback (GObject *source_object,
-                   GAsyncResult *res,
-                   gpointer user_data)
-{
-  SoupRequestHTTP *request = SOUP_REQUEST_HTTP(source_object);
-  g_autoptr(GInputStream) in = NULL;
-  LoadUriData *data = user_data;
-
-  in = soup_request_send_finish (SOUP_REQUEST (request), res, &data->error);
-  if (in == NULL)
-    {
-      g_main_loop_quit (data->loop);
-      return;
-    }
-
-  g_autoptr(SoupMessage) msg = soup_request_http_get_message ((SoupRequestHTTP*) request);
-  if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
-    {
-      int code;
-      GQuark domain = G_IO_ERROR;
-
-      switch (msg->status_code)
-        {
-        case 304:
-          //TODO          domain = FLATPAK_OCI_ERROR;
-          //TODO          code = FLATPAK_OCI_ERROR_NOT_CHANGED;
-          code = G_IO_ERROR_FAILED;
-          break;
-        case 404:
-        case 410:
-          code = G_IO_ERROR_NOT_FOUND;
-          break;
-        default:
-          code = G_IO_ERROR_FAILED;
-        }
-
-      data->error = g_error_new (domain, code,
-                                 "Server returned status %u: %s",
-                                 msg->status_code,
-                                 soup_status_get_phrase (msg->status_code));
-      g_main_loop_quit (data->loop);
-      return;
-    }
-
-  data->etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag"));
-
-  g_input_stream_read_async (in, data->buffer, sizeof (data->buffer),
-                             G_PRIORITY_DEFAULT, data->cancellable,
-                             load_uri_read_cb, data);
-}
-
 SoupSession *
 flatpak_create_soup_session (const char *user_agent)
 {
@@ -1212,52 +1099,25 @@ flatpak_create_soup_session (const char *user_agent)
   return soup_session;
 }
 
-gboolean
-flatpak_download_http_uri (SoupSession *soup_session,
-                           const char   *uri,
-                           GOutputStream *out,
-                           FlatpakLoadUriProgress progress,
-                           gpointer      user_data,
-                           GCancellable *cancellable,
-                           GError      **error)
+CURL *
+flatpak_create_curl_session (const char *user_agent)
 {
-  g_autoptr(SoupRequestHTTP) request = NULL;
-  g_autoptr(GMainLoop) loop = NULL;
-  g_autoptr(GMainContext) context = NULL;
-  LoadUriData data = { NULL };
+  CURL *curl_session;
 
-  g_debug ("Loading %s using libsoup", uri);
+  curl_global_init (CURL_GLOBAL_DEFAULT);
 
-  context = g_main_context_ref_thread_default ();
+  curl_session = curl_easy_init ();
+  if (curl_session == NULL)
+    return NULL;
 
-  loop = g_main_loop_new (context, TRUE);
-  data.loop = loop;
-  data.out = out;
-  data.progress = progress;
-  data.cancellable = cancellable;
-  data.user_data = user_data;
-  data.last_progress_time = g_get_monotonic_time ();
+  curl_easy_setopt (curl_session, CURLOPT_CONNECTTIMEOUT, 60);
+  curl_easy_setopt (curl_session, CURLOPT_FAILONERROR, 1);
+  curl_easy_setopt (curl_session, CURLOPT_FOLLOWLOCATION, 1);
+  curl_easy_setopt (curl_session, CURLOPT_MAXREDIRS, 50);
+  curl_easy_setopt (curl_session, CURLOPT_NOPROGRESS, 0);
+  curl_easy_setopt (curl_session, CURLOPT_USERAGENT, user_agent);
 
-  request = soup_session_request_http (soup_session, "GET",
-                                       uri, error);
-  if (request == NULL)
-    return FALSE;
-
-  soup_request_send_async (SOUP_REQUEST(request),
-                           cancellable,
-                           load_uri_callback, &data);
-
-  g_main_loop_run (loop);
-
-  if (data.error)
-    {
-      g_propagate_error (error, data.error);
-      return FALSE;
-    }
-
-  g_debug ("Received %" G_GUINT64_FORMAT " bytes", data.downloaded_bytes);
-
-  return TRUE;
+  return curl_session;
 }
 
 typedef enum {

src/builder-flatpak-utils.h

@@ -29,9 +29,11 @@
 #include <libsoup/soup.h>
 #include <ostree.h>
 #include <json-glib/json-glib.h>
+#include <curl/curl.h>
 
 typedef enum {
   FLATPAK_HOST_COMMAND_FLAGS_CLEAR_ENV = 1 << 0,
+  FLATPAK_HOST_COMMAND_FLAGS_WATCH_BUS = 1 << 1,
 } FlatpakHostCommandFlags;
 
 typedef void (*FlatpakLoadUriProgress) (guint64 downloaded_bytes,
@@ -61,29 +63,21 @@ typedef void (*FlatpakLoadUriProgress) (guint64 downloaded_bytes,
 #define FLATPAK_METADATA_KEY_SDK "sdk"
 #define FLATPAK_METADATA_KEY_TAGS "tags"
 
+#define FLATPAK_METADATA_GROUP_BUILD "Build"
+#define FLATPAK_METADATA_KEY_BUILD_EXTENSIONS "built-extensions"
+
 /* https://github.com/GNOME/libglnx/pull/38
  * Note by using #define rather than wrapping via a static inline, we
  * don't have to re-define attributes like G_GNUC_PRINTF.
  */
 #define flatpak_fail glnx_throw
 
-gint flatpak_strcmp0_ptr (gconstpointer a,
-                          gconstpointer b);
-
-gboolean  flatpak_has_path_prefix (const char *str,
-                                   const char *prefix);
-
 const char * flatpak_path_match_prefix (const char *pattern,
                                         const char *path);
 
 gboolean flatpak_is_in_sandbox (void);
 
-gboolean flatpak_fancy_output (void);
-
 const char * flatpak_get_arch (void);
-const char ** flatpak_get_arches (void);
-
-char ** flatpak_get_current_locale_subpaths (void);
 
 GFile *flatpak_file_new_tmp_in (GFile *dir,
                                 const char *templatename,
@@ -109,23 +103,11 @@ gboolean flatpak_splice_update_checksum (GOutputStream  *out,
                                          GCancellable   *cancellable,
                                          GError        **error);
 
-GBytes * flatpak_read_stream (GInputStream *in,
-                              gboolean      null_terminate,
-                              GError      **error);
-
 
 gboolean flatpak_has_name_prefix (const char *string,
                                   const char *name);
-gboolean flatpak_is_valid_name (const char *string,
-                                GError **error);
-gboolean flatpak_is_valid_branch (const char *string,
-                                  GError **error);
 
 char * flatpak_make_valid_id_prefix (const char *orig_id);
-gboolean flatpak_id_has_subref_suffix (const char *id);
-
-char **flatpak_decompose_ref (const char *ref,
-                              GError    **error);
 
 char * flatpak_compose_ref (gboolean    app,
                             const char *name,
@@ -194,13 +176,7 @@ flatpak_auto_lock_helper (GMutex *mutex)
   return mutex;
 }
 
-gint flatpak_mkstempat (int    dir_fd,
-                        gchar *tmpl,
-                        int    flags,
-                        int    mode);
-
 char * flatpak_quote_argv (const char *argv[]);
-gboolean flatpak_file_arg_has_suffix (const char *arg, const char *suffix);
 
 gboolean            flatpak_spawn (GFile       *dir,
                                    char       **output,
@@ -226,12 +202,6 @@ gboolean flatpak_file_rename (GFile *from,
                               GCancellable  *cancellable,
                               GError       **error);
 
-gboolean flatpak_openat_noatime (int            dfd,
-                                 const char    *name,
-                                 int           *ret_fd,
-                                 GCancellable  *cancellable,
-                                 GError       **error);
-
 typedef enum {
   FLATPAK_CP_FLAGS_NONE = 0,
   FLATPAK_CP_FLAGS_MERGE = 1<<0,
@@ -242,6 +212,7 @@ typedef enum {
 gboolean   flatpak_cp_a (GFile         *src,
                          GFile         *dest,
                          FlatpakCpFlags flags,
+                         GPtrArray     *skip_files,
                          GCancellable  *cancellable,
                          GError       **error);
 
@@ -258,13 +229,6 @@ gboolean flatpak_rm_rf (GFile         *dir,
                         GCancellable  *cancellable,
                         GError       **error);
 
-gboolean flatpak_open_in_tmpdir_at (int                tmpdir_fd,
-                                    int                mode,
-                                    char              *tmpl,
-                                    GOutputStream    **out_stream,
-                                    GCancellable      *cancellable,
-                                    GError           **error);
-
 static inline void
 flatpak_temp_dir_destroy (void *p)
 {
@@ -310,7 +274,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoCommitModifier, ostree_repo_commit_modi
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoDevInoCache, ostree_repo_devino_cache_unref)
 #endif
 
-#ifndef SOUP_AUTOCLEANUPS_H
+#if !defined(SOUP_AUTOCLEANUPS_H) && !defined(__SOUP_AUTOCLEANUPS_H__)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (SoupSession, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (SoupMessage, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (SoupRequest, g_object_unref)
@@ -345,21 +309,7 @@ gboolean flatpak_allocate_tmpdir (int           tmpdir_dfd,
 
 
 SoupSession * flatpak_create_soup_session (const char *user_agent);
-GBytes * flatpak_load_http_uri (SoupSession *soup_session,
-                                const char   *uri,
-                                const char   *etag,
-                                char        **out_etag,
-                                FlatpakLoadUriProgress progress,
-                                gpointer      user_data,
-                                GCancellable *cancellable,
-                                GError      **error);
-gboolean flatpak_download_http_uri (SoupSession *soup_session,
-                                    const char   *uri,
-                                    GOutputStream *out,
-                                    FlatpakLoadUriProgress progress,
-                                    gpointer      user_data,
-                                    GCancellable *cancellable,
-                                    GError      **error);
+CURL * flatpak_create_curl_session (const char *user_agent);
 
 typedef struct FlatpakContext FlatpakContext;
 

src/builder-git.c

@@ -531,8 +531,10 @@ builder_git_mirror_repo (const char     *repo_location,
       else
         cached_git_dir = g_object_ref (cache_mirror_dir);
 
-      /* If we're not updating, only pull from cache to avoid network i/o */
-      if (!update)
+      /* If the ref already exists (it may not with a shallow mirror
+       * if it has changed) and we're not updating, only pull from
+       * cache to avoid network i/o. */
+      if (already_exists && !update)
         {
           if (cached_git_dir)
             origin = g_file_get_uri (cached_git_dir);
@@ -821,44 +823,6 @@ git_extract_submodule (const char     *repo_location,
   return TRUE;
 }
 
-gboolean
-builder_git_checkout_dir (const char     *repo_location,
-                          const char     *branch,
-                          const char     *dir,
-                          GFile          *dest,
-                          BuilderContext *context,
-                          GError        **error)
-{
-  g_autoptr(GFile) mirror_dir = NULL;
-  g_autofree char *mirror_dir_path = NULL;
-  g_autofree char *dest_path = NULL;
-  g_autofree char *dest_path_git = NULL;
-
-  mirror_dir = git_get_mirror_dir (repo_location, context);
-
-  mirror_dir_path = g_file_get_path (mirror_dir);
-  dest_path = g_file_get_path (dest);
-  dest_path_git = g_build_filename (dest_path, ".git", NULL);
-
-  g_mkdir_with_parents (dest_path, 0755);
-
-  if (!cp (error,
-           "-al",
-           mirror_dir_path, dest_path_git, NULL))
-    return FALSE;
-
-  /* Then we need to convert to regular */
-  if (!git (dest, NULL, 0, error,
-            "config", "--bool", "core.bare", "false", NULL))
-    return FALSE;
-
-  if (!git (dest, NULL, 0, error,
-            "checkout", branch, "--", dir ? dir : ".", NULL))
-    return FALSE;
-
-  return TRUE;
-}
-
 gboolean
 builder_git_checkout (const char     *repo_location,
                       const char     *branch,

src/builder-git.h

@@ -49,12 +49,6 @@ gboolean builder_git_checkout           (const char      *repo_location,
                                          GFile           *dest,
                                          BuilderContext  *context,
                                          GError         **error);
-gboolean builder_git_checkout_dir       (const char      *repo_location,
-                                         const char      *branch,
-                                         const char      *dir,
-                                         GFile           *dest,
-                                         BuilderContext  *context,
-                                         GError         **error);
 gboolean builder_git_shallow_mirror_ref (const char     *repo_location,
                                          const char     *destination_path,
                                          const char     *ref,

src/builder-main.c

@@ -47,6 +47,7 @@ static gboolean opt_build_only;
 static gboolean opt_finish_only;
 static gboolean opt_export_only;
 static gboolean opt_show_deps;
+static gboolean opt_show_manifest;
 static gboolean opt_disable_download;
 static gboolean opt_disable_updates;
 static gboolean opt_ccache;
@@ -70,13 +71,16 @@ static char *opt_repo;
 static char *opt_subject;
 static char *opt_body;
 static char *opt_collection_id = NULL;
+static int opt_token_type = -1;
 static char *opt_gpg_homedir;
 static char **opt_key_ids;
 static char **opt_sources_dirs;
 static char **opt_sources_urls;
+static char **opt_add_tags;
+static char **opt_remove_tags;
 static int opt_jobs;
 static char *opt_mirror_screenshots_url;
-static char *opt_install_deps_from;
+static char **opt_install_deps_from;
 static gboolean opt_install_deps_only;
 static gboolean opt_user;
 static char *opt_installation;
@@ -89,6 +93,8 @@ static GOptionEntry entries[] = {
   { "version", 0, 0, G_OPTION_ARG_NONE, &opt_version, "Print version information and exit", NULL },
   { "arch", 0, 0, G_OPTION_ARG_STRING, &opt_arch, "Architecture to build for (must be host compatible)", "ARCH" },
   { "default-branch", 0, 0, G_OPTION_ARG_STRING, &opt_default_branch, "Change the default branch", "BRANCH" },
+  { "add-tag", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_add_tags, "Add a tag to the build", "TAG"},
+  { "remove-tag", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_remove_tags, "Remove a tag from the build", "TAG"},
   { "run", 0, 0, G_OPTION_ARG_NONE, &opt_run, "Run a command in the build directory (see --run --help)", NULL },
   { "ccache", 0, 0, G_OPTION_ARG_NONE, &opt_ccache, "Use ccache", NULL },
   { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Disable cache lookups", NULL },
@@ -105,15 +111,15 @@ static GOptionEntry entries[] = {
   { "export-only", 0, 0, G_OPTION_ARG_NONE, &opt_export_only, "Only run export phase", NULL },
   { "allow-missing-runtimes", 0, 0, G_OPTION_ARG_NONE, &opt_allow_missing_runtimes, "Don't fail if runtime and sdk missing", NULL },
   { "show-deps", 0, 0, G_OPTION_ARG_NONE, &opt_show_deps, "List the dependencies of the json file (see --show-deps --help)", NULL },
+  { "show-manifest", 0, 0, G_OPTION_ARG_NONE, &opt_show_manifest, "Print out the manifest file in standard json format (see --show-manifest --help)", NULL },
   { "require-changes", 0, 0, G_OPTION_ARG_NONE, &opt_require_changes, "Don't create app dir or export if no changes", NULL },
   { "keep-build-dirs", 0, 0, G_OPTION_ARG_NONE, &opt_keep_build_dirs, "Don't remove build directories after install", NULL },
   { "delete-build-dirs", 0, 0, G_OPTION_ARG_NONE, &opt_delete_build_dirs, "Always remove build directories, even after build failure", NULL },
   { "repo", 0, 0, G_OPTION_ARG_STRING, &opt_repo, "Repo to export into", "DIR"},
   { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject (passed to build-export)", "SUBJECT" },
   { "body", 'b', 0, G_OPTION_ARG_STRING, &opt_body, "Full description (passed to build-export)", "BODY" },
-#ifdef FLATPAK_ENABLE_P2P
   { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, "Collection ID (passed to build-export)", "COLLECTION-ID" },
-#endif  /* FLATPAK_ENABLE_P2P */
+  { "token-type", 0, 0, G_OPTION_ARG_INT, &opt_token_type, "Set type of token needed to install this commit (passed to build-export)", "VAL" },
   { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"},
   { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"},
   { "force-clean", 0, 0, G_OPTION_ARG_NONE, &opt_force_clean, "Erase previous contents of DIRECTORY", NULL },
@@ -127,7 +133,7 @@ static GOptionEntry entries[] = {
   { "from-git-branch", 0, 0, G_OPTION_ARG_STRING, &opt_from_git_branch, "Branch to use in --from-git", "BRANCH"},
   { "mirror-screenshots-url", 0, 0, G_OPTION_ARG_STRING, &opt_mirror_screenshots_url, "Download and rewrite screenshots to match this url", "URL"},
   { "install", 0, 0, G_OPTION_ARG_NONE, &opt_install, "Install if build succeeds", NULL},
-  { "install-deps-from", 0, 0, G_OPTION_ARG_STRING, &opt_install_deps_from, "Install build dependencies from this remote", "REMOTE"},
+  { "install-deps-from", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_install_deps_from, "Install build dependencies from this remote", "REMOTE"},
   { "install-deps-only", 0, 0, G_OPTION_ARG_NONE, &opt_install_deps_only, "Stop after installing dependencies"},
   { "user", 0, 0, G_OPTION_ARG_NONE, &opt_user, "Install dependencies in user installations", NULL },
   { "system", 0, G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &opt_user, "Install dependencies in system-wide installations (default)", NULL },
@@ -150,7 +156,13 @@ static GOptionEntry run_entries[] = {
 
 static GOptionEntry show_deps_entries[] = {
   { "verbose", 'v', 0, G_OPTION_ARG_NONE, &opt_verbose, "Print debug information during command processing", NULL },
-  { "show-deps", 0, 0, G_OPTION_ARG_NONE, &opt_show_deps, "List the dependencies of the json file (see --show-deps --help)", NULL },
+  { "show-deps", 0, 0, G_OPTION_ARG_NONE, &opt_show_deps, "List the dependencies of the json file", NULL },
+  { NULL }
+};
+
+static GOptionEntry show_manifest_entries[] = {
+  { "verbose", 'v', 0, G_OPTION_ARG_NONE, &opt_verbose, "Print debug information during command processing", NULL },
+  { "show-manifest", 0, 0, G_OPTION_ARG_NONE, &opt_show_manifest, "Print out the manifest file in standard json format", NULL },
   { NULL }
 };
 
@@ -189,6 +201,7 @@ do_export (BuilderContext *build_context,
            char          **exclude_dirs,
            const gchar    *branch,
            const gchar    *collection_id,
+           gint32          token_type,
            ...)
 {
   va_list ap;
@@ -196,7 +209,6 @@ do_export (BuilderContext *build_context,
   int i;
 
   g_autoptr(GPtrArray) args = NULL;
-  g_autoptr(GSubprocess) subp = NULL;
 
   args = g_ptr_array_new_with_free_func (g_free);
   g_ptr_array_add (args, g_strdup ("flatpak"));
@@ -222,8 +234,11 @@ do_export (BuilderContext *build_context,
   if (collection_id)
     g_ptr_array_add (args, g_strdup_printf ("--collection-id=%s", collection_id));
 
+  if (token_type >= 0)
+    g_ptr_array_add (args, g_strdup_printf ("--token-type=%d", token_type));
+
   /* Additional flags. */
-  va_start (ap, collection_id);
+  va_start (ap, token_type);
   while ((arg = va_arg (ap, const gchar *)))
     if (arg != skip_arg)
       g_ptr_array_add (args, g_strdup ((gchar *) arg));
@@ -242,16 +257,8 @@ do_export (BuilderContext *build_context,
 
   g_ptr_array_add (args, NULL);
 
-  subp =
-    g_subprocess_newv ((const gchar * const *) args->pdata,
-                       G_SUBPROCESS_FLAGS_NONE,
-                       error);
-
-  if (subp == NULL ||
-      !g_subprocess_wait_check (subp, NULL, error))
-    return FALSE;
-
-  return TRUE;
+  return flatpak_spawnv (NULL, NULL, G_SUBPROCESS_FLAGS_NONE, error,
+                         (const gchar * const *) args->pdata);
 }
 
 static gboolean
@@ -264,7 +271,6 @@ do_install (BuilderContext *build_context,
   g_autofree char *ref = NULL;
 
   g_autoptr(GPtrArray) args = NULL;
-  g_autoptr(GSubprocess) subp = NULL;
 
   args = g_ptr_array_new_with_free_func (g_free);
   g_ptr_array_add (args, g_strdup ("flatpak"));
@@ -277,9 +283,9 @@ do_install (BuilderContext *build_context,
   else
     g_ptr_array_add (args, g_strdup ("--system"));
 
-  if (opt_user)
-    g_ptr_array_add (args, g_strdup ("-y"));
-
+  g_ptr_array_add (args, g_strdup ("-y"));
+  if (flatpak_version_check (1, 2, 0))
+    g_ptr_array_add (args, g_strdup ("--noninteractive"));
   g_ptr_array_add (args, g_strdup ("--reinstall"));
 
   ref = flatpak_build_untyped_ref (id, branch,
@@ -290,16 +296,72 @@ do_install (BuilderContext *build_context,
 
   g_ptr_array_add (args, NULL);
 
-  subp =
-    g_subprocess_newv ((const gchar * const *) args->pdata,
-                       G_SUBPROCESS_FLAGS_NONE,
-                       error);
+  return flatpak_spawnv (NULL, NULL, G_SUBPROCESS_FLAGS_NONE, error,
+                         (const gchar * const *) args->pdata);
+}
 
-  if (subp == NULL ||
-      !g_subprocess_wait_check (subp, NULL, error))
-    return FALSE;
+static gboolean
+git (char   **output,
+     GError **error,
+     ...)
+{
+  gboolean res;
+  va_list ap;
 
-  return TRUE;
+  if (output != NULL)
+    *output = NULL;
+
+  va_start (ap, error);
+  res = flatpak_spawn (NULL, output, 0, error, "git", ap);
+  va_end (ap);
+
+  if (output != NULL &&
+      (*output != NULL && *output[0] == '\0'))
+    {
+      g_free (*output);
+      *output = NULL;
+    }
+
+  return res;
+}
+
+static char *
+trim_linefeed (char *str)
+{
+  guint len;
+
+  g_return_val_if_fail (str != NULL, NULL);
+
+  len = strlen (str);
+  str[len] = '\0';
+
+  return str;
+}
+
+static void
+git_init_email (void)
+{
+  char *user, *email;
+
+  /* Have an email for author and committer */
+  if (!git (&email, NULL, "config", "--get", "user.email", NULL) ||
+      email == NULL)
+    email = g_strdup ("flatpak-builder-commit@flatpak.org");
+  else
+    email = trim_linefeed (email);
+  g_setenv ("GIT_AUTHOR_EMAIL", email, FALSE);
+  g_setenv ("GIT_COMMITTER_EMAIL", email, FALSE);
+  g_free (email);
+
+  /* Have a "real name" for author and committer */
+  if (!git (&user, NULL, "config", "--get", "user.name", NULL) ||
+      user == NULL)
+    user = g_strdup ("Flatpak git committer");
+  else
+    user = trim_linefeed (user);
+  g_setenv ("GIT_AUTHOR_NAME", user, FALSE);
+  g_setenv ("GIT_COMMITTER_NAME", user, FALSE);
+  g_free (user);
 }
 
 int
@@ -312,9 +374,9 @@ main (int    argc,
   g_autoptr(BuilderManifest) manifest = NULL;
   g_autoptr(GOptionContext) context = NULL;
   const char *app_dir_path = NULL, *manifest_rel_path;
-  g_autofree gchar *json = NULL;
-  g_autofree gchar *json_sha256 = NULL;
-  g_autofree gchar *old_json_sha256 = NULL;
+  g_autofree gchar *manifest_contents = NULL;
+  g_autofree gchar *manifest_sha256 = NULL;
+  g_autofree gchar *old_manifest_sha256 = NULL;
   g_autoptr(BuilderContext) build_context = NULL;
   g_autoptr(GFile) base_dir = NULL;
   g_autoptr(GFile) manifest_file = NULL;
@@ -331,7 +393,9 @@ main (int    argc,
   g_autofree char **orig_argv = NULL;
   gboolean is_run = FALSE;
   gboolean is_show_deps = FALSE;
+  gboolean is_show_manifest = FALSE;
   gboolean app_dir_is_empty = FALSE;
+  gboolean prune_unused_stages = FALSE;
   g_autoptr(FlatpakContext) arg_context = NULL;
   g_autoptr(FlatpakTempDir) cleanup_manifest_dir = NULL;
   g_autofree char *manifest_basename = NULL;
@@ -339,6 +403,7 @@ main (int    argc,
   int i, first_non_arg, orig_argc;
   int argnr;
   char *p;
+  struct stat statbuf;
 
   setlocale (LC_ALL, "");
 
@@ -355,6 +420,14 @@ main (int    argc,
   else
     g_unsetenv ("GIO_USE_VFS");
 
+  /* Work around libsoup/glib race condition, as per:
+     https://bugzilla.gnome.org/show_bug.cgi?id=796031 and
+     https://bugzilla.gnome.org/show_bug.cgi?id=674885#c87 */
+  g_type_ensure (G_TYPE_SOCKET_FAMILY);
+  g_type_ensure (G_TYPE_SOCKET_TYPE);
+  g_type_ensure (G_TYPE_SOCKET_PROTOCOL);
+  g_type_ensure (G_TYPE_SOCKET_ADDRESS);
+
   orig_argv = g_memdup (argv, sizeof (char *) * argc);
   orig_argc = argc;
 
@@ -368,6 +441,8 @@ main (int    argc,
         is_run = TRUE;
       if (strcmp (argv[i], "--show-deps") == 0)
         is_show_deps = TRUE;
+      if (strcmp (argv[i], "--show-manifest") == 0)
+        is_show_manifest = TRUE;
     }
 
   if (is_run)
@@ -385,6 +460,11 @@ main (int    argc,
       context = g_option_context_new ("MANIFEST - Show manifest dependencies");
       g_option_context_add_main_entries (context, show_deps_entries, NULL);
     }
+  else if (is_show_manifest)
+    {
+      context = g_option_context_new ("MANIFEST - Show manifest");
+      g_option_context_add_main_entries (context, show_manifest_entries, NULL);
+    }
   else
     {
       context = g_option_context_new ("DIRECTORY MANIFEST - Build manifest");
@@ -408,7 +488,7 @@ main (int    argc,
 
   argnr = 1;
 
-  if (!is_show_deps)
+  if (!is_show_deps && !is_show_manifest)
     {
       if (argc == argnr)
         return usage (context, "DIRECTORY must be specified");
@@ -420,14 +500,22 @@ main (int    argc,
   manifest_rel_path = argv[argnr++];
   manifest_basename = g_path_get_basename (manifest_rel_path);
 
-#ifdef FLATPAK_ENABLE_P2P
   if (opt_collection_id != NULL &&
       !ostree_validate_collection_id (opt_collection_id, &error))
     {
       g_printerr ("‘%s’ is not a valid collection ID: %s", opt_collection_id, error->message);
       return 1;
     }
-#endif  /* FLATPAK_ENABLE_P2P */
+
+  if (opt_token_type < -1
+#if G_MAXINT > 0x7fffffff
+      || opt_token_type > G_MAXINT32
+#endif
+      )
+    {
+      g_printerr ("--token-type value must be a 32 bit integer >= 0\n");
+      return 1;
+    }
 
   if (app_dir_path)
     app_dir = g_file_new_for_path (app_dir_path);
@@ -446,6 +534,8 @@ main (int    argc,
   builder_context_set_rebuild_on_sdk_change (build_context, opt_rebuild_on_sdk_change);
   builder_context_set_bundle_sources (build_context, opt_bundle_sources);
 
+  git_init_email ();
+
   if (opt_sources_dirs)
     {
       g_autoptr(GPtrArray) sources_dirs = NULL;
@@ -464,6 +554,12 @@ main (int    argc,
       sources_urls = g_ptr_array_new_with_free_func ((GDestroyNotify)soup_uri_free);
       for (i = 0; opt_sources_urls[i] != NULL; i++)
         {
+          if (!g_str_has_suffix (opt_sources_urls[i], "/"))
+            {
+              g_autofree gchar *tmp = opt_sources_urls[i];
+              opt_sources_urls[i] = g_strdup_printf ("%s/", tmp);
+            }
+
           SoupURI *uri = soup_uri_new (opt_sources_urls[i]);
           if (uri == NULL)
             {
@@ -505,21 +601,27 @@ main (int    argc,
 
       cleanup_manifest_dir = g_object_ref (build_subdir);
 
+      int mirror_flags = FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
+
+      if (opt_disable_updates)
+        {
+          mirror_flags |= FLATPAK_GIT_MIRROR_FLAGS_UPDATE;
+        }
+
       if (!builder_git_mirror_repo (opt_from_git,
                                     NULL,
-                                    opt_disable_updates?0:FLATPAK_GIT_MIRROR_FLAGS_UPDATE,
+                                    mirror_flags,
                                     git_branch, build_context, &error))
         {
           g_printerr ("Can't clone manifest repo: %s\n", error->message);
           return 1;
         }
 
-      if (!builder_git_checkout_dir (opt_from_git,
-                                     git_branch,
-                                     manifest_dirname,
-                                     build_subdir,
-                                     build_context,
-                                     &error))
+      if (!builder_git_checkout (opt_from_git,
+                                 git_branch,
+                                 build_subdir,
+                                 build_context,
+                                 &error))
         {
           g_printerr ("Can't check out manifest repo: %s\n", error->message);
           return 1;
@@ -536,18 +638,21 @@ main (int    argc,
 
   builder_context_set_base_dir (build_context, base_dir);
 
-  if (!g_file_get_contents (flatpak_file_get_path_cached (manifest_file), &json, NULL, &error))
+  if (!g_file_get_contents (flatpak_file_get_path_cached (manifest_file), &manifest_contents, NULL, &error))
     {
       g_printerr ("Can't load '%s': %s\n", manifest_rel_path, error->message);
       return 1;
     }
 
-  json_sha256 = g_compute_checksum_for_string (G_CHECKSUM_SHA256, json, -1);
+  if (stat (flatpak_file_get_path_cached (manifest_file), &statbuf) == 0)
+    builder_context_set_source_date_epoch (build_context, (gint64)statbuf.st_mtime);
+
+  manifest_sha256 = g_compute_checksum_for_string (G_CHECKSUM_SHA256, manifest_contents, -1);
 
   if (opt_skip_if_unchanged)
     {
-      old_json_sha256 = builder_context_get_checksum_for (build_context, manifest_basename);
-      if (old_json_sha256 != NULL && strcmp (json_sha256, old_json_sha256) == 0)
+      old_manifest_sha256 = builder_context_get_checksum_for (build_context, manifest_basename);
+      if (old_manifest_sha256 != NULL && strcmp (manifest_sha256, old_manifest_sha256) == 0)
         {
           g_print ("No changes to manifest, skipping\n");
           return 42;
@@ -557,8 +662,8 @@ main (int    argc,
   /* Can't push this as user data to the demarshalling :/ */
   builder_manifest_set_demarshal_base_dir (builder_context_get_base_dir (build_context));
 
-  manifest = (BuilderManifest *) json_gobject_from_data (BUILDER_TYPE_MANIFEST,
-                                                         json, -1, &error);
+  manifest = (BuilderManifest *) builder_gobject_from_data (BUILDER_TYPE_MANIFEST, manifest_rel_path,
+                                                            manifest_contents, &error);
 
   builder_manifest_set_demarshal_base_dir (NULL);
 
@@ -568,12 +673,21 @@ main (int    argc,
       return 1;
     }
 
+  if (opt_remove_tags)
+    builder_manifest_remove_tags (manifest, (const char **)opt_remove_tags);
+
+  if (opt_add_tags)
+    builder_manifest_add_tags (manifest, (const char **)opt_add_tags);
+
   if (opt_default_branch)
-    builder_manifest_set_default_branch (manifest, opt_default_branch);
+    builder_context_set_default_branch (build_context, opt_default_branch);
 
   if (opt_collection_id)
     builder_manifest_set_default_collection_id (manifest, opt_collection_id);
 
+  if (opt_token_type >= 0)
+    builder_manifest_set_default_token_type (manifest, (gint32)opt_token_type);
+
   if (is_run && argc == 3)
     return usage (context, "Program to run must be specified");
 
@@ -587,19 +701,26 @@ main (int    argc,
     {
       if (!builder_manifest_show_deps (manifest, build_context, &error))
         {
-          g_printerr ("Error running %s: %s\n", argv[3], error->message);
+          g_printerr ("Error calculating deps: %s\n", error->message);
           return 1;
         }
 
       return 0;
     }
 
+  if (is_show_manifest)
+    {
+      g_autofree char *json = builder_manifest_serialize (manifest);
+      g_print ("%s\n", json);
+      return 0;
+    }
+
   if (opt_install_deps_from != NULL)
     {
       if (!builder_manifest_install_deps (manifest, build_context, opt_install_deps_from, opt_user, opt_installation,
                                           opt_yes, &error))
         {
-          g_printerr ("Error running %s: %s\n", argv[3], error->message);
+          g_printerr ("Error installing deps: %s\n", error->message);
           return 1;
         }
       if (opt_install_deps_only)
@@ -667,28 +788,29 @@ main (int    argc,
     }
 
   /* Verify that cache and build dir is on same filesystem */
-  {
-    g_autofree char *state_path = g_file_get_path (builder_context_get_state_dir (build_context));
-    g_autoptr(GFile) app_parent = g_file_get_parent (builder_context_get_app_dir (build_context));
-    g_autofree char *app_parent_path = g_file_get_path (app_parent);
-    struct stat buf1, buf2;
+  if (!opt_download_only)
+    {
+      g_autofree char *state_path = g_file_get_path (builder_context_get_state_dir (build_context));
+      g_autoptr(GFile) app_parent = g_file_get_parent (builder_context_get_app_dir (build_context));
+      g_autofree char *app_parent_path = g_file_get_path (app_parent);
+      struct stat buf1, buf2;
 
-    if (stat (app_parent_path, &buf1) == 0 && stat (state_path, &buf2) == 0 &&
-        buf1.st_dev != buf2.st_dev)
-      {
-        g_printerr ("The state dir (%s) is not on the same filesystem as the target dir (%s)\n",
-                    state_path, app_parent_path);
-        return 1;
-      }
-  }
+      if (stat (app_parent_path, &buf1) == 0 && stat (state_path, &buf2) == 0 &&
+          buf1.st_dev != buf2.st_dev)
+        {
+          g_printerr ("The state dir (%s) is not on the same filesystem as the target dir (%s)\n",
+                      state_path, app_parent_path);
+          return 1;
+        }
+    }
 
-  if (!builder_context_set_checksum_for (build_context, manifest_basename, json_sha256, &error))
+  if (!builder_context_set_checksum_for (build_context, manifest_basename, manifest_sha256, &error))
     {
       g_printerr ("Failed to set checksum for ‘%s’: %s\n", manifest_basename, error->message);
       return 1;
     }
 
-  if (!builder_manifest_start (manifest, opt_allow_missing_runtimes, build_context, &error))
+  if (!builder_manifest_start (manifest, opt_download_only, opt_allow_missing_runtimes, build_context, &error))
     {
       g_printerr ("Failed to init: %s\n", error->message);
       return 1;
@@ -793,7 +915,7 @@ main (int    argc,
         }
 
       if (builder_context_get_bundle_sources (build_context) &&
-          !builder_manifest_bundle_sources (manifest, json, cache, build_context, &error))
+          !builder_manifest_bundle_sources (manifest, manifest_contents, cache, build_context, &error))
         {
           g_printerr ("Error: %s\n", error->message);
           return 1;
@@ -806,14 +928,24 @@ main (int    argc,
   if (opt_mirror_screenshots_url && !opt_export_only)
     {
       g_autofree char *screenshot_subdir = g_strdup_printf ("%s-%s", builder_manifest_get_id (manifest),
-                                                            builder_manifest_get_branch (manifest));
+                                                            builder_manifest_get_branch (manifest, build_context));
       g_autofree char *url = g_build_filename (opt_mirror_screenshots_url, screenshot_subdir, NULL);
       g_autofree char *xml_relpath = g_strdup_printf ("files/share/app-info/xmls/%s.xml.gz", builder_manifest_get_id (manifest));
       g_autoptr(GFile) xml = g_file_resolve_relative_path (app_dir, xml_relpath);
       g_autoptr(GFile) cache = flatpak_build_file (builder_context_get_state_dir (build_context), "screenshots-cache", NULL);
       g_autoptr(GFile) screenshots = flatpak_build_file (app_dir, "screenshots", NULL);
       g_autoptr(GFile) screenshots_sub = flatpak_build_file (screenshots, screenshot_subdir, NULL);
+      g_autofree char *fs_app_dir = g_strdup_printf ("--filesystem=%s", flatpak_file_get_path_cached (app_dir));
+      g_autofree char *fs_cache = g_strdup_printf ("--filesystem=%s", flatpak_file_get_path_cached (cache));
       const char *argv[] = {
+        "flatpak",
+        "build",
+        "--die-with-parent",
+        "--nofilesystem=host",
+        fs_app_dir,
+        fs_cache,
+        "--share=network",
+        flatpak_file_get_path_cached (app_dir),
         "appstream-util",
         "mirror-screenshots",
         flatpak_file_get_path_cached (xml),
@@ -834,6 +966,16 @@ main (int    argc,
 
       if (g_file_query_exists (xml, NULL))
         {
+          if (!flatpak_mkdir_p (cache, NULL, &error))
+            {
+              g_printerr ("Error creating screenshot cache dir: %s\n", error->message);
+              return 1;
+            }
+          if (!flatpak_break_hardlink (xml, &error))
+            {
+              g_printerr ("Error mirroring screenshots: %s\n", error->message);
+              return 1;
+            }
           if (!builder_maybe_host_spawnv (NULL,
                                           NULL,
                                           0,
@@ -868,8 +1010,9 @@ main (int    argc,
       if (!do_export (build_context, &error,
                       FALSE,
                       flatpak_file_get_path_cached (export_repo),
-                      app_dir_path, exclude_dirs, builder_manifest_get_branch (manifest),
+                      app_dir_path, exclude_dirs, builder_manifest_get_branch (manifest, build_context),
                       builder_manifest_get_collection_id (manifest),
+                      builder_manifest_get_token_type (manifest),
                       "--exclude=/lib/debug/*",
                       "--include=/lib/debug/app",
                       builder_context_get_separate_locales (build_context) ? "--exclude=/share/runtime/locale/*/*" : skip_arg,
@@ -902,8 +1045,9 @@ main (int    argc,
                                    "/share/runtime/locale/", NULL);
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           metadata_arg,
                           files_arg,
                           NULL))
@@ -922,8 +1066,9 @@ main (int    argc,
 
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           "--metadata=metadata.debuginfo",
                           builder_context_get_build_runtime (build_context) ? "--files=usr/lib/debug" : "--files=files/lib/debug",
                           NULL))
@@ -953,8 +1098,9 @@ main (int    argc,
 
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           metadata_arg, files_arg,
                           NULL))
             {
@@ -972,8 +1118,9 @@ main (int    argc,
 
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           "--metadata=metadata.sources",
                           "--files=sources",
                           NULL))
@@ -992,8 +1139,9 @@ main (int    argc,
 
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           "--metadata=metadata.platform",
                           "--files=platform",
                           builder_context_get_separate_locales (build_context) ? "--exclude=/share/runtime/locale/*/*" : skip_arg,
@@ -1026,8 +1174,9 @@ main (int    argc,
           files_arg = g_strconcat ("--files=platform/share/runtime/locale/", NULL);
           if (!do_export (build_context, &error, TRUE,
                           flatpak_file_get_path_cached (export_repo),
-                          app_dir_path, NULL, builder_manifest_get_branch (manifest),
+                          app_dir_path, NULL, builder_manifest_get_branch (manifest, build_context),
                           builder_manifest_get_collection_id (manifest),
+                          builder_manifest_get_token_type (manifest),
                           metadata_arg,
                           files_arg,
                           NULL))
@@ -1046,7 +1195,7 @@ main (int    argc,
         g_printerr ("NOTE: No export due to --require-changes, ignoring --install\n");
       else if (!do_install (build_context, flatpak_file_get_path_cached (export_repo),
                             builder_manifest_get_id (manifest),
-                            builder_manifest_get_branch (manifest),
+                            builder_manifest_get_branch (manifest, build_context),
                             &error))
         {
           g_printerr ("Install failed: %s\n", error->message);
@@ -1054,7 +1203,10 @@ main (int    argc,
         }
     }
 
-  if (!builder_gc (cache, &error))
+  if (!opt_finish_only && !opt_export_only)
+    prune_unused_stages = TRUE;
+
+  if (!builder_gc (cache, prune_unused_stages, &error))
     {
       g_warning ("Failed to GC build cache: %s", error->message);
       g_clear_error (&error);

src/builder-manifest.h

@@ -40,7 +40,7 @@ typedef struct BuilderManifest BuilderManifest;
 /* Bump this if format changes in incompatible ways to force rebuild */
 #define BUILDER_MANIFEST_CHECKSUM_VERSION "5"
 #define BUILDER_MANIFEST_CHECKSUM_CLEANUP_VERSION "1"
-#define BUILDER_MANIFEST_CHECKSUM_FINISH_VERSION "2"
+#define BUILDER_MANIFEST_CHECKSUM_FINISH_VERSION "3"
 #define BUILDER_MANIFEST_CHECKSUM_BUNDLE_SOURCES_VERSION "1"
 #define BUILDER_MANIFEST_CHECKSUM_PLATFORM_VERSION "1"
 
@@ -58,17 +58,27 @@ char *          builder_manifest_get_locale_id_platform (BuilderManifest *self);
 BuilderOptions *builder_manifest_get_build_options (BuilderManifest *self);
 GList *         builder_manifest_get_modules (BuilderManifest *self);
 GList *         builder_manifest_get_add_extensions (BuilderManifest *self);
-const char *    builder_manifest_get_branch (BuilderManifest *self);
-void            builder_manifest_set_default_branch (BuilderManifest *self,
-                                                     const char *default_branch);
+GList *         builder_manifest_get_add_build_extensions (BuilderManifest *self);
+const char *    builder_manifest_get_branch (BuilderManifest *self,
+                                             BuilderContext  *context);
 const char *    builder_manifest_get_collection_id (BuilderManifest *self);
+gint32          builder_manifest_get_token_type (BuilderManifest *self);
+const char *    builder_manifest_get_extension_tag (BuilderManifest *self);
 void            builder_manifest_set_default_collection_id (BuilderManifest *self,
                                                             const char      *default_collection_id);
+void            builder_manifest_set_default_token_type (BuilderManifest *self,
+                                                         gint32           default_token_type);
+
+void            builder_manifest_add_tags (BuilderManifest *self,
+                                           const char      **add_tags);
+void            builder_manifest_remove_tags (BuilderManifest *self,
+                                              const char      **remove_tags);
 
 
 char **         builder_manifest_get_exclude_dirs (BuilderManifest *self);
 
 gboolean        builder_manifest_start (BuilderManifest *self,
+                                        gboolean         download_only,
                                         gboolean         allow_missing_runtimes,
                                         BuilderContext  *context,
                                         GError         **error);
@@ -91,7 +101,7 @@ gboolean        builder_manifest_build (BuilderManifest *self,
                                         GError         **error);
 gboolean        builder_manifest_install_deps (BuilderManifest *self,
                                                BuilderContext  *context,
-                                               const char *remote,
+                                               char * const *remotes,
                                                gboolean opt_user,
                                                const char *opt_installation,
                                                gboolean opt_yes,
@@ -127,6 +137,7 @@ gboolean        builder_manifest_create_platform (BuilderManifest *self,
                                                   BuilderCache    *cache,
                                                   BuilderContext  *context,
                                                   GError         **error);
+char *          builder_manifest_serialize       (BuilderManifest *self);
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderManifest, g_object_unref)
 

src/builder-module.c

@@ -341,6 +341,8 @@ builder_module_set_property (GObject      *object,
 
     case PROP_CMAKE:
       self->cmake = g_value_get_boolean (value);
+      if (self->cmake)
+        g_printerr ("The cmake module property is deprecated, use buildsystem cmake or cmake-ninja instead.\n");
       break;
 
     case PROP_BUILDSYSTEM:
@@ -719,10 +721,10 @@ builder_module_serialize_property (JsonSerializable *serializable,
     }
   else
     {
-      return json_serializable_default_serialize_property (serializable,
-                                                           property_name,
-                                                           value,
-                                                           pspec);
+      return builder_serializable_serialize_property (serializable,
+                                                      property_name,
+                                                      value,
+                                                      pspec);
     }
 }
 
@@ -747,7 +749,7 @@ load_sources_from_json (const char *sources_relpath)
     }
 
   builder_manifest_set_demarshal_base_dir (sources_file_dir);
-  sources_root = json_from_string (sources_json, &error);
+  sources_root = builder_json_node_from_data (sources_path, sources_json, &error);
   if (sources_root == NULL)
     {
       g_printerr ("Error parsing %s: %s\n", sources_relpath, error->message);
@@ -824,14 +826,14 @@ builder_module_deserialize_property (JsonSerializable *serializable,
                   g_autoptr(GFile) module_file =
                     g_file_resolve_relative_path (saved_demarshal_base_dir, module_relpath);
                   const char *module_path = flatpak_file_get_path_cached (module_file);
-                  g_autofree char *json = NULL;
+                  g_autofree char *module_contents = NULL;
 
-                  if (g_file_get_contents (module_path, &json, NULL, NULL))
+                  if (g_file_get_contents (module_path, &module_contents, NULL, NULL))
                     {
                       g_autoptr(GFile) module_file_dir = g_file_get_parent (module_file);
                       builder_manifest_set_demarshal_base_dir (module_file_dir);
-                      module = json_gobject_from_data (BUILDER_TYPE_MODULE,
-                                                       json, -1, NULL);
+                      module = builder_gobject_from_data (BUILDER_TYPE_MODULE,
+                                                          module_relpath, module_contents, NULL);
                       builder_manifest_set_demarshal_base_dir (saved_demarshal_base_dir);
                       if (module)
                         {
@@ -910,10 +912,10 @@ builder_module_deserialize_property (JsonSerializable *serializable,
     }
   else
     {
-      return json_serializable_default_deserialize_property (serializable,
-                                                             property_name,
-                                                             value,
-                                                             pspec, property_node);
+      return builder_serializable_deserialize_property (serializable,
+                                                        property_name,
+                                                        value,
+                                                        pspec, property_node);
     }
 }
 
@@ -922,7 +924,10 @@ serializable_iface_init (JsonSerializableIface *serializable_iface)
 {
   serializable_iface->serialize_property = builder_module_serialize_property;
   serializable_iface->deserialize_property = builder_module_deserialize_property;
-  serializable_iface->find_property = builder_serializable_find_property_with_error;
+  serializable_iface->find_property = builder_serializable_find_property;
+  serializable_iface->list_properties = builder_serializable_list_properties;
+  serializable_iface->set_property = builder_serializable_set_property;
+  serializable_iface->get_property = builder_serializable_get_property;
 }
 
 const char *
@@ -931,6 +936,14 @@ builder_module_get_name (BuilderModule *self)
   return self->name;
 }
 
+void
+builder_module_set_name (BuilderModule *self,
+                         const char *name)
+{
+  g_free (self->name);
+  self->name = g_strdup (name);
+}
+
 gboolean
 builder_module_is_enabled (BuilderModule *self,
                            BuilderContext *context)
@@ -1162,6 +1175,7 @@ setup_build_args (GFile          *app_dir,
   else
     builddir = "/run/build/";
 
+  g_ptr_array_add (args, g_strdup_printf ("--env=FLATPAK_BUILDER_BUILDDIR=%s%s", builddir, module_name));
   g_ptr_array_add (args, g_strdup ("--nofilesystem=host"));
 
   /* We mount the canonical location, because bind-mounts of symlinks don't really work */
@@ -1378,6 +1392,7 @@ builder_module_build_helper (BuilderModule  *self,
   GFile *app_dir = builder_context_get_app_dir (context);
   g_autofree char *make_j = NULL;
   g_autofree char *make_l = NULL;
+  g_autofree char *n_jobs = NULL;
   const char *make_cmd = NULL;
   const char *test_arg = NULL;
 
@@ -1425,6 +1440,9 @@ builder_module_build_helper (BuilderModule  *self,
   env = builder_options_get_env (self->build_options, context);
   config_opts = builder_options_get_config_opts (self->build_options, context, self->config_opts);
 
+  n_jobs = g_strdup_printf ("%d", self->no_parallel_make ? 1 : builder_context_get_jobs (context));
+  env = g_environ_setenv (env, "FLATPAK_BUILDER_N_JOBS", n_jobs, FALSE);
+
   if (!self->buildsystem)
     {
       if (self->cmake)
@@ -1557,6 +1575,8 @@ builder_module_build_helper (BuilderModule  *self,
       g_auto(GStrv) configure_args = NULL;
       g_autoptr(GPtrArray) configure_args_arr = g_ptr_array_new ();
       g_autofree char *configure_content = NULL;
+      const char *prefix = NULL;
+      const char *libdir = NULL;
 
       if (!g_file_load_contents (configure_file, NULL, &configure_content, NULL, NULL, error))
         {
@@ -1629,22 +1649,27 @@ builder_module_build_helper (BuilderModule  *self,
       else if (cmake_ninja)
         cmake_generator = "Ninja";
 
+      prefix = builder_options_get_prefix (self->build_options, context);
+      libdir = builder_options_get_libdir (self->build_options, context);
+
       if (cmake || cmake_ninja)
         {
-          g_ptr_array_add (configure_args_arr, g_strdup_printf ("-DCMAKE_INSTALL_PREFIX:PATH='%s'",
-                                                                builder_options_get_prefix (self->build_options, context)));
+          g_ptr_array_add (configure_args_arr, g_strdup_printf ("-DCMAKE_INSTALL_PREFIX:PATH='%s'", prefix));
+          if (libdir)
+            g_ptr_array_add (configure_args_arr, g_strdup_printf ("-DCMAKE_INSTALL_LIBDIR:PATH='%s'", libdir));
           g_ptr_array_add (configure_args_arr, g_strdup ("-G"));
           g_ptr_array_add (configure_args_arr, g_strdup_printf ("%s", cmake_generator));
         }
       else if (qmake)
         {
-          g_ptr_array_add (configure_args_arr, g_strdup_printf ("PREFIX='%s'",
-                                                                builder_options_get_prefix (self->build_options, context)));
+          g_ptr_array_add (configure_args_arr, g_strdup_printf ("PREFIX='%s'", prefix));
+          /* TODO: What parameter for qmake? */
         }
       else /* autotools and meson */
         {
-          g_ptr_array_add (configure_args_arr, g_strdup_printf ("--prefix=%s",
-                                                                builder_options_get_prefix (self->build_options, context)));
+          g_ptr_array_add (configure_args_arr, g_strdup_printf ("--prefix=%s", prefix));
+          if (libdir)
+            g_ptr_array_add (configure_args_arr, g_strdup_printf ("--libdir=%s", libdir));
         }
 
       g_ptr_array_add (configure_args_arr, configure_final_arg);
@@ -1694,6 +1719,11 @@ builder_module_build_helper (BuilderModule  *self,
       make_j = g_strdup_printf ("-j%d", builder_context_get_jobs (context));
       make_l = g_strdup_printf ("-l%d", 2 * builder_context_get_jobs (context));
     }
+  else if (meson || cmake_ninja)
+    {
+      /* ninja defaults to a parallel make, disable it if requested */
+      make_j = g_strdup ("-j1");
+    }
 
   if (run_shell)
     {
@@ -1988,9 +2018,9 @@ builder_module_checksum_for_cleanup (BuilderModule  *self,
 }
 
 void
-builder_module_checksum_for_platform (BuilderModule  *self,
-                                      BuilderCache   *cache,
-                                      BuilderContext *context)
+builder_module_checksum_for_platform_cleanup (BuilderModule  *self,
+                                              BuilderCache   *cache,
+                                              BuilderContext *context)
 {
   builder_cache_checksum_strv (cache, self->cleanup_platform);
 }

src/builder-module.h

@@ -40,6 +40,8 @@ typedef struct BuilderModule BuilderModule;
 GType builder_module_get_type (void);
 
 const char * builder_module_get_name (BuilderModule *self);
+void         builder_module_set_name (BuilderModule *self,
+                                      const char *name);
 gboolean     builder_module_is_enabled (BuilderModule *self,
                                         BuilderContext *context);
 gboolean     builder_module_get_disabled (BuilderModule *self);
@@ -89,9 +91,9 @@ void     builder_module_checksum (BuilderModule  *self,
 void     builder_module_checksum_for_cleanup (BuilderModule  *self,
                                               BuilderCache   *cache,
                                               BuilderContext *context);
-void     builder_module_checksum_for_platform (BuilderModule  *self,
-                                               BuilderCache   *cache,
-                                               BuilderContext *context);
+void     builder_module_checksum_for_platform_cleanup (BuilderModule  *self,
+                                                       BuilderCache   *cache,
+                                                       BuilderContext *context);
 void     builder_module_cleanup_collect (BuilderModule  *self,
                                          gboolean        platform,
                                          BuilderContext *context,

src/builder-options.c

@@ -40,9 +40,13 @@ struct BuilderOptions
   gboolean    no_debuginfo;
   gboolean    no_debuginfo_compression;
   char       *cflags;
+  gboolean    cflags_override;
   char       *cppflags;
+  gboolean    cppflags_override;
   char       *cxxflags;
+  gboolean    cxxflags_override;
   char       *ldflags;
+  gboolean    ldflags_override;
   char       *append_path;
   char       *prepend_path;
   char       *append_ld_library_path;
@@ -50,6 +54,7 @@ struct BuilderOptions
   char       *append_pkg_config_path;
   char       *prepend_pkg_config_path;
   char       *prefix;
+  char       *libdir;
   char      **env;
   char      **build_args;
   char      **test_args;
@@ -72,10 +77,15 @@ G_DEFINE_TYPE_WITH_CODE (BuilderOptions, builder_options, G_TYPE_OBJECT,
 enum {
   PROP_0,
   PROP_CFLAGS,
+  PROP_CFLAGS_OVERRIDE,
   PROP_CPPFLAGS,
+  PROP_CPPFLAGS_OVERRIDE,
   PROP_CXXFLAGS,
+  PROP_CXXFLAGS_OVERRIDE,
   PROP_LDFLAGS,
+  PROP_LDFLAGS_OVERRIDE,
   PROP_PREFIX,
+  PROP_LIBDIR,
   PROP_ENV,
   PROP_STRIP,
   PROP_NO_DEBUGINFO,
@@ -112,6 +122,7 @@ builder_options_finalize (GObject *object)
   g_free (self->append_pkg_config_path);
   g_free (self->prepend_pkg_config_path);
   g_free (self->prefix);
+  g_free (self->libdir);
   g_strfreev (self->env);
   g_strfreev (self->build_args);
   g_strfreev (self->test_args);
@@ -137,18 +148,34 @@ builder_options_get_property (GObject    *object,
       g_value_set_string (value, self->cflags);
       break;
 
+    case PROP_CFLAGS_OVERRIDE:
+      g_value_set_boolean (value, self->cflags_override);
+      break;
+
     case PROP_CPPFLAGS:
       g_value_set_string (value, self->cppflags);
       break;
 
+    case PROP_CPPFLAGS_OVERRIDE:
+      g_value_set_boolean (value, self->cppflags_override);
+      break;
+
     case PROP_CXXFLAGS:
       g_value_set_string (value, self->cxxflags);
       break;
 
+    case PROP_CXXFLAGS_OVERRIDE:
+      g_value_set_boolean (value, self->cxxflags_override);
+      break;
+
     case PROP_LDFLAGS:
       g_value_set_string (value, self->ldflags);
       break;
 
+    case PROP_LDFLAGS_OVERRIDE:
+      g_value_set_boolean (value, self->ldflags_override);
+      break;
+
     case PROP_APPEND_PATH:
       g_value_set_string (value, self->append_path);
       break;
@@ -177,6 +204,10 @@ builder_options_get_property (GObject    *object,
       g_value_set_string (value, self->prefix);
       break;
 
+    case PROP_LIBDIR:
+      g_value_set_string (value, self->libdir);
+      break;
+
     case PROP_ENV:
       g_value_set_boxed (value, self->env);
       break;
@@ -238,21 +269,37 @@ builder_options_set_property (GObject      *object,
       self->cflags = g_value_dup_string (value);
       break;
 
+    case PROP_CFLAGS_OVERRIDE:
+      self->cflags_override = g_value_get_boolean (value);
+      break;
+
     case PROP_CXXFLAGS:
       g_clear_pointer (&self->cxxflags, g_free);
       self->cxxflags = g_value_dup_string (value);
       break;
 
+    case PROP_CXXFLAGS_OVERRIDE:
+      self->cxxflags_override = g_value_get_boolean (value);
+      break;
+
     case PROP_CPPFLAGS:
       g_clear_pointer (&self->cppflags, g_free);
       self->cppflags = g_value_dup_string (value);
       break;
 
+    case PROP_CPPFLAGS_OVERRIDE:
+      self->cppflags_override = g_value_get_boolean (value);
+      break;
+
     case PROP_LDFLAGS:
       g_clear_pointer (&self->ldflags, g_free);
       self->ldflags = g_value_dup_string (value);
       break;
 
+    case PROP_LDFLAGS_OVERRIDE:
+      self->ldflags_override = g_value_get_boolean (value);
+      break;
+
     case PROP_APPEND_PATH:
       g_clear_pointer (&self->append_path, g_free);
       self->append_path = g_value_dup_string (value);
@@ -288,6 +335,11 @@ builder_options_set_property (GObject      *object,
       self->prefix = g_value_dup_string (value);
       break;
 
+    case PROP_LIBDIR:
+      g_clear_pointer (&self->libdir, g_free);
+      self->libdir = g_value_dup_string (value);
+      break;
+
     case PROP_ENV:
       tmp = self->env;
       self->env = g_strdupv (g_value_get_boxed (value));
@@ -363,6 +415,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CFLAGS_OVERRIDE,
+                                   g_param_spec_boolean ("cflags-override",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_CXXFLAGS,
                                    g_param_spec_string ("cxxflags",
@@ -370,6 +429,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CXXFLAGS_OVERRIDE,
+                                   g_param_spec_boolean ("cxxflags-override",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_CPPFLAGS,
                                    g_param_spec_string ("cppflags",
@@ -377,6 +443,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CPPFLAGS_OVERRIDE,
+                                   g_param_spec_boolean ("cppflags-override",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_LDFLAGS,
                                    g_param_spec_string ("ldflags",
@@ -384,6 +457,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_LDFLAGS_OVERRIDE,
+                                   g_param_spec_boolean ("ldflags-override",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_APPEND_PATH,
                                    g_param_spec_string ("append-path",
@@ -433,6 +513,13 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_LIBDIR,
+                                   g_param_spec_string ("libdir",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_ENV,
                                    g_param_spec_boxed ("env",
@@ -503,6 +590,7 @@ builder_options_class_init (BuilderOptionsClass *klass)
                                                          "",
                                                          FALSE,
                                                          G_PARAM_READWRITE));
+
 }
 
 static void
@@ -557,23 +645,24 @@ builder_options_serialize_property (JsonSerializable *serializable,
 
           for (i = 0; self->env[i] != NULL; i++)
             {
-              JsonNode *str = json_node_new (JSON_NODE_VALUE);
+              JsonNode *val = NULL;
               const char *equal;
               g_autofree char *member = NULL;
 
               equal = strchr (self->env[i], '=');
               if (equal)
                 {
-                  json_node_set_string (str, equal + 1);
+                  val = json_node_new (JSON_NODE_VALUE);
+                  json_node_set_string (val, equal + 1);
                   member = g_strndup (self->env[i], equal - self->env[i]);
                 }
               else
                 {
-                  json_node_set_string (str, "");
+                  val = json_node_new (JSON_NODE_NULL);
                   member = g_strdup (self->env[i]);
                 }
 
-              json_object_set_member (object, member, str);
+              json_object_set_member (object, member, val);
             }
 
           retval = json_node_init_object (json_node_alloc (), object);
@@ -655,11 +744,19 @@ builder_options_deserialize_property (JsonSerializable *serializable,
               const char *val_str;
 
               val = json_object_get_member (object, member_name);
-              val_str = json_node_get_string (val);
-              if (val_str == NULL)
-                return FALSE;
 
-              g_ptr_array_add (env, g_strdup_printf ("%s=%s", member_name, val_str));
+              if (JSON_NODE_TYPE (val) == JSON_NODE_NULL)
+                {
+                  g_ptr_array_add (env, g_strdup_printf ("%s", member_name));
+                }
+              else
+                {
+                  val_str = json_node_get_string (val);
+                  if (val_str == NULL)
+                    return FALSE;
+
+                  g_ptr_array_add (env, g_strdup_printf ("%s=%s", member_name, val_str));
+                }
             }
 
           g_ptr_array_add (env, NULL);
@@ -683,7 +780,8 @@ serializable_iface_init (JsonSerializableIface *serializable_iface)
 {
   serializable_iface->serialize_property = builder_options_serialize_property;
   serializable_iface->deserialize_property = builder_options_deserialize_property;
-  serializable_iface->find_property = builder_serializable_find_property_with_error;
+  serializable_iface->find_property = builder_serializable_find_property;
+  serializable_iface->list_properties = builder_serializable_list_properties;
 }
 
 static GList *
@@ -718,12 +816,21 @@ get_all_options (BuilderOptions *self, BuilderContext *context)
 }
 
 static const char *
-builder_options_get_flags (BuilderOptions *self, BuilderContext *context, size_t field_offset)
+builder_options_get_flags (BuilderOptions *self,
+                           BuilderContext *context,
+                           size_t          field_offset,
+                           size_t          override_field_offset,
+                           const char     *sdk_flags)
 {
   g_autoptr(GList) options = get_all_options (self, context);
   GList *l;
   GString *flags = NULL;
 
+  if (sdk_flags && sdk_flags[0])
+    {
+      flags = g_string_new (sdk_flags);
+    }
+
   /* Last command flag wins, so reverse order */
   options = g_list_reverse (options);
 
@@ -731,6 +838,10 @@ builder_options_get_flags (BuilderOptions *self, BuilderContext *context, size_t
     {
       BuilderOptions *o = l->data;
       const char *flag = G_STRUCT_MEMBER (const char *, o, field_offset);
+      gboolean override = G_STRUCT_MEMBER (gboolean, o, override_field_offset);
+
+      if (override && flags)
+        g_string_truncate (flags, 0);
 
       if (flag)
         {
@@ -750,28 +861,45 @@ builder_options_get_flags (BuilderOptions *self, BuilderContext *context, size_t
   return NULL;
 }
 
+static const char *
+get_sdk_flags (BuilderOptions *self, BuilderContext *context, const char *(*method)(BuilderSdkConfig *self))
+{
+  BuilderSdkConfig *sdk_config = builder_context_get_sdk_config (context);
+  if (sdk_config)
+    return (*method) (sdk_config);
+  return NULL;
+}
+
 const char *
 builder_options_get_cflags (BuilderOptions *self, BuilderContext *context)
 {
-  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cflags));
+  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cflags),
+                                    G_STRUCT_OFFSET (BuilderOptions, cflags_override),
+                                    get_sdk_flags (self, context, builder_sdk_config_get_cflags));
 }
 
 const char *
 builder_options_get_cxxflags (BuilderOptions *self, BuilderContext *context)
 {
-  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cxxflags));
+  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cxxflags),
+                                    G_STRUCT_OFFSET (BuilderOptions, cxxflags_override),
+                                    get_sdk_flags (self, context, builder_sdk_config_get_cxxflags));
 }
 
 const char *
 builder_options_get_cppflags (BuilderOptions *self, BuilderContext *context)
 {
-  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cppflags));
+  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, cppflags),
+                                    G_STRUCT_OFFSET (BuilderOptions, cppflags_override),
+                                    get_sdk_flags (self, context, builder_sdk_config_get_cppflags));
 }
 
 const char *
 builder_options_get_ldflags (BuilderOptions *self, BuilderContext *context)
 {
-  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, ldflags));
+  return builder_options_get_flags (self, context, G_STRUCT_OFFSET (BuilderOptions, ldflags),
+                                    G_STRUCT_OFFSET (BuilderOptions, ldflags_override),
+                                    get_sdk_flags (self, context, builder_sdk_config_get_ldflags));
 }
 
 static char *
@@ -889,6 +1017,25 @@ builder_options_get_prefix (BuilderOptions *self, BuilderContext *context)
   return "/app";
 }
 
+const char *
+builder_options_get_libdir (BuilderOptions *self, BuilderContext *context)
+{
+  g_autoptr(GList) options = get_all_options (self, context);
+  GList *l;
+
+  for (l = options; l != NULL; l = l->next)
+    {
+      BuilderOptions *o = l->data;
+      if (o->libdir)
+        return o->libdir;
+    }
+
+  if (builder_context_get_build_runtime (context))
+    return get_sdk_flags (self, context, builder_sdk_config_get_libdir);
+
+  return NULL;
+}
+
 gboolean
 builder_options_get_strip (BuilderOptions *self, BuilderContext *context)
 {
@@ -946,35 +1093,7 @@ builder_options_get_env (BuilderOptions *self, BuilderContext *context)
   char **envp = NULL;
   const char *cflags, *cppflags, *cxxflags, *ldflags;
 
-  for (l = options; l != NULL; l = l->next)
-    {
-      BuilderOptions *o = l->data;
-
-      if (o->env)
-        {
-          for (i = 0; o->env[i] != NULL; i++)
-            {
-              const char *line = o->env[i];
-              const char *eq = strchr (line, '=');
-              const char *value = "";
-              g_autofree char *key = NULL;
-
-              if (eq)
-                {
-                  key = g_strndup (line, eq - line);
-                  value = eq + 1;
-                }
-              else
-                {
-                  key = g_strdup (key);
-                }
-
-              envp = g_environ_setenv (envp, key, value, FALSE);
-            }
-        }
-    }
-
-  envp = builder_context_extend_env (context, envp);
+  envp = builder_context_extend_env_pre (context, envp);
 
   cflags = builder_options_get_cflags (self, context);
   if (cflags)
@@ -992,6 +1111,36 @@ builder_options_get_env (BuilderOptions *self, BuilderContext *context)
   if (ldflags)
     envp = g_environ_setenv (envp, "LDFLAGS", ldflags, FALSE);
 
+  /* We traverse in reverse order because the list is "last first" */
+  for (l = g_list_last (options); l != NULL; l = l->prev)
+    {
+      BuilderOptions *o = l->data;
+
+      if (o->env)
+        {
+          for (i = g_strv_length (o->env) - 1; i >= 0; i--)
+            {
+              const char *line = o->env[i];
+              const char *eq = strchr (line, '=');
+
+              if (eq)
+                {
+                  g_autofree char *key = g_strndup (line, eq - line);
+                  const char *value = eq + 1;
+
+                  envp = g_environ_setenv (envp, key, value, TRUE);
+                }
+              else
+                {
+                  envp = g_environ_unsetenv (envp, line);
+                }
+
+            }
+        }
+    }
+
+  envp = builder_context_extend_env_post (context, envp);
+
   envp = builder_options_update_path (self, context, envp);
   envp = builder_options_update_ld_path (self, context, envp);
   envp = builder_options_update_pkg_config_path (self, context, envp);
@@ -1023,10 +1172,17 @@ builder_options_get_build_args (BuilderOptions *self,
         }
     }
 
-  if (array->len > 0 && builder_context_get_sandboxed (context))
+  if (builder_context_get_sandboxed (context))
     {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't specify build-args in sandboxed build");
-      return NULL;
+      if (array->len > 0)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't specify build-args in sandboxed build");
+          return NULL;
+        }
+      /* If, for whatever reason, the app has network access in the
+         metadata, explicitly neuter that if we're building
+         sandboxed */
+      g_ptr_array_add (array, g_strdup ("--unshare=network"));
     }
 
   g_ptr_array_add (array, NULL);
@@ -1143,10 +1299,15 @@ builder_options_checksum (BuilderOptions *self,
 
   builder_cache_checksum_str (cache, BUILDER_OPTION_CHECKSUM_VERSION);
   builder_cache_checksum_str (cache, self->cflags);
+  builder_cache_checksum_compat_boolean (cache, self->cflags_override);
   builder_cache_checksum_str (cache, self->cxxflags);
+  builder_cache_checksum_compat_boolean (cache, self->cxxflags_override);
   builder_cache_checksum_str (cache, self->cppflags);
+  builder_cache_checksum_compat_boolean (cache, self->cppflags_override);
   builder_cache_checksum_str (cache, self->ldflags);
+  builder_cache_checksum_compat_boolean (cache, self->ldflags_override);
   builder_cache_checksum_str (cache, self->prefix);
+  builder_cache_checksum_compat_str (cache, self->libdir);
   builder_cache_checksum_strv (cache, self->env);
   builder_cache_checksum_strv (cache, self->build_args);
   builder_cache_checksum_compat_strv (cache, self->test_args);

src/builder-options.h

@@ -47,6 +47,8 @@ const char *builder_options_get_ldflags (BuilderOptions *self,
                                          BuilderContext *context);
 const char *builder_options_get_prefix (BuilderOptions *self,
                                         BuilderContext *context);
+const char *builder_options_get_libdir (BuilderOptions *self,
+                                        BuilderContext *context);
 char **     builder_options_get_env (BuilderOptions *self,
                                      BuilderContext *context);
 char **     builder_options_get_build_args (BuilderOptions *self,

src/builder-post-process.c

@@ -112,6 +112,9 @@ invalidate_old_python_compiled (const char *path,
   return TRUE;
 }
 
+/* We need to read at least 12 bytes to get both magic + optional flags + mtime */
+#define PYTHON_HEADER_SIZE 12
+
 static gboolean
 fixup_python_time_stamp (const char *path,
                          const char *rel_path,
@@ -119,9 +122,11 @@ fixup_python_time_stamp (const char *path,
 {
   glnx_fd_close int fd = -1;
   g_auto(GLnxTmpfile) tmpf = { 0 };
-  guint8 buffer[8];
+  guint8 buffer[PYTHON_HEADER_SIZE];
   ssize_t res;
   guint32 pyc_mtime;
+  guint32 magic, header_flag;
+  gsize mtime_offset;
   g_autofree char *py_path = NULL;
   struct stat stbuf;
   gboolean remove_pyc = FALSE;
@@ -136,8 +141,8 @@ fixup_python_time_stamp (const char *path,
       return TRUE;
     }
 
-  res = pread (fd, buffer, 8, 0);
-  if (res != 8)
+  res = pread (fd, buffer, PYTHON_HEADER_SIZE, 0);
+  if (res != PYTHON_HEADER_SIZE)
     {
       g_warning ("Short read for %s", rel_path);
       return TRUE;
@@ -149,11 +154,48 @@ fixup_python_time_stamp (const char *path,
       return TRUE;
     }
 
+  magic = buffer[0] + (buffer[1] << 8);
+
+  /* All magic listed here: https://github.com/python/cpython/blob/HEAD/Lib/importlib/_bootstrap_external.py#L167
+   * 3392 is the first (3.7) which added an extra flags field in the header.
+   * 20121 is the first higher major listed (1.5), and all other non-py3 ones are higher.
+   */
+  if (magic >= 3392 && magic < 20121)
+    {
+      /* From the spec:
+       * The pyc header currently consists of 3 32-bit words. We will expand it to 4.
+       * The first word will continue to be the magic number, versioning the bytecode and pyc format.
+       * The second word, conceptually the new word, will be a bit field.
+       * The interpretation of the rest of the header and invalidation behavior of the pyc depends on the contents of the bit field.
+       */
+      header_flag =
+        (buffer[4] << 8*0) |
+        (buffer[5] << 8*1) |
+        (buffer[6] << 8*2) |
+        (buffer[7] << 8*3);
+
+      /* If the bit field is 0, the pyc is a traditional timestamp-based pyc. I.e., the third
+         and forth words will be the timestamp and file size respectively, and invalidation
+         will be done by comparing the metadata of the source file  with that in the header. */
+      if (header_flag != 0)
+        {
+          /* Non-mtime based verification, like hash if low bit is 1,
+           * or other future added methods.  No need to do anything*/
+          return TRUE;
+        }
+
+      mtime_offset = 8;
+    }
+  else
+    {
+      mtime_offset = 4;
+    }
+
   pyc_mtime =
-    (buffer[4] << 8*0) |
-    (buffer[5] << 8*1) |
-    (buffer[6] << 8*2) |
-    (buffer[7] << 8*3);
+    (buffer[mtime_offset+0] << 8*0) |
+    (buffer[mtime_offset+1] << 8*1) |
+    (buffer[mtime_offset+2] << 8*2) |
+    (buffer[mtime_offset+3] << 8*3);
 
   if (strcmp (dir_basename, "__pycache__") == 0)
     {
@@ -233,11 +275,11 @@ fixup_python_time_stamp (const char *path,
     return glnx_throw_errno_prefix (error, "copyfile");
 
   /* Change to mtime 0 which is what ostree uses for checkouts */
-  buffer[4] = OSTREE_TIMESTAMP;
-  buffer[5] = buffer[6] = buffer[7] = 0;
+  buffer[mtime_offset+0] = OSTREE_TIMESTAMP;
+  buffer[mtime_offset+1] = buffer[mtime_offset+2] = buffer[mtime_offset+3] = 0;
 
-  res = pwrite (tmpf.fd, buffer, 8, 0);
-  if (res != 8)
+  res = pwrite (tmpf.fd, buffer, PYTHON_HEADER_SIZE, 0);
+  if (res != PYTHON_HEADER_SIZE)
     {
       glnx_set_error_from_errno (error);
       return FALSE;

src/builder-sdk-config.c

@@ -0,0 +1,236 @@
+/*
+ * Copyright © 2018 Codethink Limited
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Valentin David <valentin.david@codethink.co.uk>
+ */
+
+#include "builder-sdk-config.h"
+
+#include <json-glib/json-glib.h>
+
+struct BuilderSdkConfig {
+  GObject       parent;
+
+  char         *libdir;
+  char         *cppflags;
+  char         *cflags;
+  char         *cxxflags;
+  char         *ldflags;
+};
+
+typedef struct
+{
+  GObjectClass parent_class;
+} BuilderSdkConfigClass;
+
+G_DEFINE_TYPE (BuilderSdkConfig, builder_sdk_config, G_TYPE_OBJECT)
+
+static void
+builder_sdk_config_finalize (GObject *object)
+{
+  BuilderSdkConfig *self = (BuilderSdkConfig *) object;
+
+  g_free (self->libdir);
+  g_free (self->cppflags);
+  g_free (self->cflags);
+  g_free (self->cxxflags);
+  g_free (self->ldflags);
+
+  G_OBJECT_CLASS (builder_sdk_config_parent_class)->finalize (object);
+}
+
+enum {
+  PROP_0,
+  PROP_LIBDIR,
+  PROP_CPPFLAGS,
+  PROP_CFLAGS,
+  PROP_CXXFLAGS,
+  PROP_LDFLAGS,
+  LAST_PROP
+};
+
+static void
+builder_sdk_config_get_property (GObject    *object,
+                                 guint       prop_id,
+                                 GValue     *value,
+                                 GParamSpec *pspec)
+{
+  BuilderSdkConfig *self = BUILDER_SDK_CONFIG (object);
+
+  switch (prop_id)
+    {
+    case PROP_LIBDIR:
+      g_value_set_string (value, self->libdir);
+      break;
+
+    case PROP_CPPFLAGS:
+      g_value_set_string (value, self->cppflags);
+      break;
+
+    case PROP_CFLAGS:
+      g_value_set_string (value, self->cflags);
+      break;
+
+    case PROP_CXXFLAGS:
+      g_value_set_string (value, self->cxxflags);
+      break;
+
+    case PROP_LDFLAGS:
+      g_value_set_string (value, self->ldflags);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+builder_sdk_config_set_property (GObject      *object,
+                                 guint         prop_id,
+                                 const GValue *value,
+                                 GParamSpec   *pspec)
+{
+  BuilderSdkConfig *self = BUILDER_SDK_CONFIG (object);
+
+  switch (prop_id)
+    {
+    case PROP_LIBDIR:
+      g_free (self->libdir);
+      self->libdir = g_value_dup_string(value);
+      break ;
+
+    case PROP_CPPFLAGS:
+      g_free (self->cppflags);
+      self->cppflags = g_value_dup_string(value);
+      break ;
+
+    case PROP_CFLAGS:
+      g_free (self->cflags);
+      self->cflags = g_value_dup_string(value);
+      break ;
+
+    case PROP_CXXFLAGS:
+      g_free (self->cxxflags);
+      self->cxxflags = g_value_dup_string(value);
+      break ;
+
+    case PROP_LDFLAGS:
+      g_free (self->ldflags);
+      self->ldflags = g_value_dup_string(value);
+      break ;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+builder_sdk_config_class_init (BuilderSdkConfigClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->finalize = builder_sdk_config_finalize;
+  object_class->get_property = builder_sdk_config_get_property;
+  object_class->set_property = builder_sdk_config_set_property;
+
+  g_object_class_install_property (object_class,
+                                   PROP_LIBDIR,
+                                   g_param_spec_string ("libdir",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CPPFLAGS,
+                                   g_param_spec_string ("cppflags",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CFLAGS,
+                                   g_param_spec_string ("cflags",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_CXXFLAGS,
+                                   g_param_spec_string ("cxxflags",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_LDFLAGS,
+                                   g_param_spec_string ("ldflags",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+}
+
+static void
+builder_sdk_config_init (BuilderSdkConfig *self)
+{
+}
+
+
+const char *
+builder_sdk_config_get_libdir (BuilderSdkConfig *self)
+{
+  return self->libdir;
+}
+
+const char *
+builder_sdk_config_get_cppflags (BuilderSdkConfig *self)
+{
+  return self->cppflags;
+}
+
+const char *
+builder_sdk_config_get_cflags (BuilderSdkConfig *self)
+{
+  return self->cflags;
+}
+
+const char *
+builder_sdk_config_get_cxxflags (BuilderSdkConfig *self)
+{
+  return self->cxxflags;
+}
+
+const char *
+builder_sdk_config_get_ldflags (BuilderSdkConfig *self)
+{
+  return self->ldflags;
+}
+
+BuilderSdkConfig *
+builder_sdk_config_from_file (GFile    *file,
+                              GError  **error)
+{
+  g_autofree gchar     *config_contents = NULL;
+  gsize                 config_size;
+
+  if (!g_file_load_contents (file, NULL, &config_contents, &config_size, NULL, error))
+    return NULL;
+
+  return (BuilderSdkConfig*) json_gobject_from_data (BUILDER_TYPE_SDK_CONFIG,
+                                                     config_contents,
+                                                     config_size,
+                                                     error);
+}

src/builder-sdk-config.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright © 2018 Codethink Limited
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Valentin David <valentin.david@codethink.co.uk>
+ */
+
+#ifndef __BUILDER_SDK_CONFIG_H__
+#define __BUILDER_SDK_CONFIG_H__
+
+#include <glib-object.h>
+#include <gio/gio.h>
+
+G_BEGIN_DECLS
+
+typedef struct BuilderSdkConfig BuilderSdkConfig;
+
+#define BUILDER_TYPE_SDK_CONFIG (builder_sdk_config_get_type ())
+#define BUILDER_SDK_CONFIG(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), BUILDER_TYPE_SDK_CONFIG, BuilderSdkConfig))
+
+GType builder_sdk_config_get_type (void);
+
+const char *      builder_sdk_config_get_libdir   (BuilderSdkConfig  *self);
+const char *      builder_sdk_config_get_cppflags (BuilderSdkConfig  *self);
+const char *      builder_sdk_config_get_cflags   (BuilderSdkConfig  *self);
+const char *      builder_sdk_config_get_cxxflags (BuilderSdkConfig  *self);
+const char *      builder_sdk_config_get_ldflags  (BuilderSdkConfig  *self);
+
+BuilderSdkConfig *builder_sdk_config_from_file    (GFile             *file,
+                                                   GError           **error);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderSdkConfig, g_object_unref)
+
+G_END_DECLS
+
+#endif /* __BUILDER_SDK_CONFIG_H__ */

src/builder-source-archive.c

@@ -38,11 +38,15 @@ struct BuilderSourceArchive
 
   char         *path;
   char         *url;
+  char        **mirror_urls;
   char         *md5;
   char         *sha1;
   char         *sha256;
   char         *sha512;
   guint         strip_components;
+  char         *dest_filename;
+  gboolean      git_init;
+  char         *archive_type;
 };
 
 typedef struct
@@ -61,6 +65,10 @@ enum {
   PROP_SHA256,
   PROP_SHA512,
   PROP_STRIP_COMPONENTS,
+  PROP_DEST_FILENAME,
+  PROP_MIRROR_URLS,
+  PROP_GIT_INIT,
+  PROP_ARCHIVE_TYPE,
   LAST_PROP
 };
 
@@ -75,7 +83,8 @@ typedef enum {
   TAR_LZMA,
   TAR_LZOP,
   TAR_XZ,
-  ZIP
+  ZIP,
+  SEVENZ,
 } BuilderArchiveType;
 
 static gboolean
@@ -127,6 +136,9 @@ builder_source_archive_finalize (GObject *object)
   g_free (self->sha1);
   g_free (self->sha256);
   g_free (self->sha512);
+  g_free (self->dest_filename);
+  g_strfreev (self->mirror_urls);
+  g_free (self->archive_type);
 
   G_OBJECT_CLASS (builder_source_archive_parent_class)->finalize (object);
 }
@@ -169,6 +181,22 @@ builder_source_archive_get_property (GObject    *object,
       g_value_set_uint (value, self->strip_components);
       break;
 
+    case PROP_DEST_FILENAME:
+      g_value_set_string (value, self->dest_filename);
+      break;
+
+    case PROP_MIRROR_URLS:
+      g_value_set_boxed (value, self->mirror_urls);
+      break;
+
+    case PROP_GIT_INIT:
+      g_value_set_boolean (value, self->git_init);
+      break;
+
+    case PROP_ARCHIVE_TYPE:
+      g_value_set_string (value, self->archive_type);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -181,6 +209,7 @@ builder_source_archive_set_property (GObject      *object,
                                      GParamSpec   *pspec)
 {
   BuilderSourceArchive *self = BUILDER_SOURCE_ARCHIVE (object);
+  gchar **tmp;
 
   switch (prop_id)
     {
@@ -218,6 +247,26 @@ builder_source_archive_set_property (GObject      *object,
       self->strip_components = g_value_get_uint (value);
       break;
 
+    case PROP_DEST_FILENAME:
+      g_free (self->dest_filename);
+      self->dest_filename = g_value_dup_string (value);
+      break;
+
+    case PROP_MIRROR_URLS:
+      tmp = self->mirror_urls;
+      self->mirror_urls = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
+    case PROP_GIT_INIT:
+      self->git_init = g_value_get_boolean (value);
+      break;
+
+    case PROP_ARCHIVE_TYPE:
+      g_free (self->archive_type);
+      self->archive_type = g_value_dup_string (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -263,7 +312,10 @@ get_download_location (BuilderSourceArchive *self,
 
   path = soup_uri_get_path (uri);
 
-  base_name = g_path_get_basename (path);
+  if (self->dest_filename)
+    base_name = g_strdup (self->dest_filename);
+  else
+    base_name = g_path_get_basename (path);
 
   builder_get_all_checksums (checksums, checksums_type,
                              self->md5,
@@ -358,21 +410,10 @@ builder_source_archive_download (BuilderSource  *source,
 
   if (g_file_query_exists (file, NULL))
     {
-      if (is_local && checksums[0]  != NULL)
-        {
-          g_autofree char *data = NULL;
-          gsize len;
-
-          if (!g_file_load_contents (file, NULL, &data, &len, NULL, error))
-            return FALSE;
-
-          if (!builder_verify_checksums (base_name,
-                                         data, len,
-                                         checksums, checksums_type,
-                                         error))
-            return FALSE;
-        }
-      return TRUE;
+      return !is_local || checksums[0] == NULL ||
+             builder_verify_checksums (base_name, file,
+                                       checksums, checksums_type,
+                                       error);
     }
 
   if (is_local)
@@ -383,6 +424,7 @@ builder_source_archive_download (BuilderSource  *source,
 
   if (!builder_context_download_uri (context,
                                      self->url,
+                                     (const char **)self->mirror_urls,
                                      file,
                                      checksums,
                                      checksums_type,
@@ -422,6 +464,19 @@ unzip (GFile   *dir,
   return res;
 }
 
+static gboolean
+un7z (GFile       *dir,
+      const char  *sevenz_path,
+      GError     **error)
+{
+  gboolean res;
+  const gchar *argv[] = { "7z",  "x", sevenz_path, NULL };
+
+  res = flatpak_spawnv (dir, NULL, 0, error, argv);
+
+  return res;
+}
+
 static gboolean
 unrpm (GFile   *dir,
        const char *rpm_path,
@@ -475,7 +530,8 @@ get_type (GFile *archivefile)
   if (g_str_has_suffix (lower, ".tar.lzo"))
     return TAR_LZOP;
 
-  if (g_str_has_suffix (lower, ".tar.xz"))
+  if (g_str_has_suffix (lower, ".tar.xz") ||
+      g_str_has_suffix (lower, ".txz"))
     return TAR_XZ;
 
   if (g_str_has_suffix (lower, ".zip"))
@@ -566,6 +622,73 @@ create_uncompress_directory (BuilderSourceArchive *self, GFile *dest, GError **e
   return uncompress_dest;
 }
 
+static gboolean
+git (GFile   *dir,
+     GError **error,
+     ...)
+{
+  gboolean res;
+  va_list ap;
+
+  va_start (ap, error);
+  res = flatpak_spawn (dir, NULL, 0, error, "git", ap);
+  va_end (ap);
+
+  return res;
+}
+
+static gboolean
+init_git (GFile   *dir,
+          GError **error)
+{
+  char *basename;
+
+  basename = g_file_get_basename (dir);
+  if (!git (dir, error, "init", NULL) ||
+      !git (dir, error, "add", "--ignore-errors", ".", NULL) ||
+      !git (dir, error, "commit", "-m", basename, NULL))
+    {
+      g_free (basename);
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+static BuilderArchiveType
+get_type_from_prop (BuilderSourceArchive *self)
+{
+  struct {
+    const char *id;
+    BuilderArchiveType type;
+  } str_to_types[] = {
+      { "rpm", RPM },
+      { "tar", TAR },
+      { "tar-gzip", TAR_GZIP },
+      { "tar-compress", TAR_COMPRESS },
+      { "tar-bzip2", TAR_BZIP2 },
+      { "tar-lzip", TAR_LZIP },
+      { "tar-lzma", TAR_LZMA },
+      { "tar-xz", TAR_XZ },
+      { "zip", ZIP },
+      { "7z", SEVENZ },
+  };
+  guint i;
+
+  if (self->archive_type == NULL)
+    return UNKNOWN;
+
+  for (i = 0; i < G_N_ELEMENTS(str_to_types); i++)
+    {
+      if (g_strcmp0 (self->archive_type, str_to_types[i].id) == 0)
+        return str_to_types[i].type;
+    }
+
+  g_warning ("Unknown archive-type \"%s\"", self->archive_type);
+
+  return UNKNOWN;
+}
+
 static gboolean
 builder_source_archive_extract (BuilderSource  *source,
                                 GFile          *dest,
@@ -584,7 +707,9 @@ builder_source_archive_extract (BuilderSource  *source,
   if (archivefile == NULL)
     return FALSE;
 
-  type = get_type (archivefile);
+  type = get_type_from_prop (self);
+  if (type == UNKNOWN)
+    type = get_type (archivefile);
 
   archive_path = g_file_get_path (archivefile);
 
@@ -612,6 +737,23 @@ builder_source_archive_extract (BuilderSource  *source,
             return FALSE;
         }
     }
+  else if (type == SEVENZ)
+    {
+      g_autoptr(GFile) sevenz_dest = NULL;
+
+      sevenz_dest = create_uncompress_directory (self, dest, error);
+      if (sevenz_dest == NULL)
+        return FALSE;
+
+      if (!un7z (sevenz_dest, archive_path, error))
+        return FALSE;
+
+      if (self->strip_components > 0)
+        {
+          if (!strip_components_into (dest, sevenz_dest, self->strip_components, error))
+            return FALSE;
+        }
+    }
   else if (type == RPM)
     {
       g_autoptr(GFile) rpm_dest = NULL;
@@ -635,6 +777,12 @@ builder_source_archive_extract (BuilderSource  *source,
       return FALSE;
     }
 
+  if (self->git_init)
+    {
+      if (!init_git (dest, error))
+        return FALSE;
+    }
+
   return TRUE;
 }
 
@@ -705,6 +853,8 @@ builder_source_archive_checksum (BuilderSource  *source,
   builder_cache_checksum_compat_str (cache, self->sha1);
   builder_cache_checksum_compat_str (cache, self->sha512);
   builder_cache_checksum_uint32 (cache, self->strip_components);
+  builder_cache_checksum_compat_str (cache, self->dest_filename);
+  builder_cache_checksum_compat_strv (cache, self->mirror_urls);
 }
 
 
@@ -774,6 +924,34 @@ builder_source_archive_class_init (BuilderSourceArchiveClass *klass)
                                                       0, G_MAXUINT,
                                                       1,
                                                       G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_DEST_FILENAME,
+                                   g_param_spec_string ("dest-filename",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_MIRROR_URLS,
+                                   g_param_spec_boxed ("mirror-urls",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_GIT_INIT,
+                                   g_param_spec_boolean ("git-init",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_ARCHIVE_TYPE,
+                                   g_param_spec_string ("archive-type",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
 }
 
 static void

src/builder-source-bzr.c

@@ -120,11 +120,14 @@ bzr (GFile   *dir,
      GError **error,
      ...)
 {
+  g_autofree char *brz = NULL;
   gboolean res;
   va_list ap;
 
+  brz = g_find_program_in_path ("brz");
+
   va_start (ap, error);
-  res = flatpak_spawn (dir, output, 0, error, "bzr", ap);
+  res = flatpak_spawn (dir, output, 0, error, brz ? brz : "bzr", ap);
   va_end (ap);
 
   return res;

src/builder-source-dir.c

@@ -0,0 +1,305 @@
+/* builder-source-dir.c
+ *
+ * Copyright (C) 2015 Red Hat, Inc
+ *
+ * This file is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Alexander Larsson <alexl@redhat.com>
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/statfs.h>
+
+#include "builder-flatpak-utils.h"
+
+#include "builder-utils.h"
+#include "builder-source-dir.h"
+
+struct BuilderSourceDir
+{
+  BuilderSource parent;
+
+  char         *path;
+  char        **skip;
+};
+
+typedef struct
+{
+  BuilderSourceClass parent_class;
+} BuilderSourceDirClass;
+
+G_DEFINE_TYPE (BuilderSourceDir, builder_source_dir, BUILDER_TYPE_SOURCE);
+
+enum {
+  PROP_0,
+  PROP_PATH,
+  PROP_SKIP,
+  LAST_PROP
+};
+
+static void
+builder_source_dir_finalize (GObject *object)
+{
+  BuilderSourceDir *self = (BuilderSourceDir *) object;
+
+  g_free (self->path);
+  g_strfreev (self->skip);
+
+  G_OBJECT_CLASS (builder_source_dir_parent_class)->finalize (object);
+}
+
+static void
+builder_source_dir_get_property (GObject    *object,
+                                 guint       prop_id,
+                                 GValue     *value,
+                                 GParamSpec *pspec)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (object);
+
+  switch (prop_id)
+    {
+    case PROP_PATH:
+      g_value_set_string (value, self->path);
+      break;
+
+    case PROP_SKIP:
+      g_value_set_boxed (value, self->skip);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+builder_source_dir_set_property (GObject      *object,
+                                 guint         prop_id,
+                                 const GValue *value,
+                                 GParamSpec   *pspec)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (object);
+  gchar **tmp;
+
+  switch (prop_id)
+    {
+    case PROP_PATH:
+      g_free (self->path);
+      self->path = g_value_dup_string (value);
+      break;
+
+    case PROP_SKIP:
+      tmp = self->skip;
+      self->skip = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static GFile *
+get_source_file (BuilderSourceDir *self,
+                 BuilderContext    *context,
+                 GError           **error)
+{
+  GFile *base_dir = BUILDER_SOURCE (self)->base_dir;
+
+  if (self->path != NULL && self->path[0] != 0)
+    {
+      return g_file_resolve_relative_path (base_dir, self->path);
+    }
+
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "source dir path not specified");
+  return NULL;
+}
+
+static gboolean
+builder_source_dir_show_deps (BuilderSource  *source,
+                              GError        **error)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (source);
+
+  if (self->path && self->path[0] != 0)
+    g_print ("%s\n", self->path);
+
+  return TRUE;
+}
+
+static gboolean
+builder_source_dir_download (BuilderSource  *source,
+                             gboolean        update_vcs,
+                             BuilderContext *context,
+                             GError        **error)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (source);
+  g_autoptr(GFile) file = NULL;
+
+  file = get_source_file (self, context, error);
+  if (file == NULL)
+    return FALSE;
+
+  if (g_file_query_file_type (file, G_FILE_QUERY_INFO_NONE, NULL) != G_FILE_TYPE_DIRECTORY)
+    return flatpak_fail (error, "Can't find directory at %s", self->path);
+
+  return TRUE;
+}
+
+static GPtrArray *
+builder_source_dir_get_skip (BuilderSource  *source,
+                             BuilderContext *context)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (source);
+  g_autoptr(GPtrArray) skip = g_ptr_array_new_with_free_func (g_object_unref);
+  int i;
+
+  g_ptr_array_add (skip, g_object_ref (builder_context_get_app_dir_raw (context)));
+  g_ptr_array_add (skip, g_object_ref (builder_context_get_state_dir (context)));
+
+  if (self->skip)
+    {
+      g_autoptr(GFile) source_dir = get_source_file (self, context, NULL);
+
+      for (i = 0; source_dir != NULL && self->skip[i] != NULL; i++)
+        {
+          GFile *f = g_file_resolve_relative_path (source_dir, self->skip[i]);
+          if (f)
+            g_ptr_array_add (skip, f);
+        }
+    }
+
+  return g_steal_pointer (&skip);
+}
+
+static gboolean
+builder_source_dir_extract (BuilderSource  *source,
+                            GFile          *dest,
+                            BuilderOptions *build_options,
+                            BuilderContext *context,
+                            GError        **error)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (source);
+  g_autoptr(GFile) src = NULL;
+  g_autoptr(GPtrArray) skip = NULL;
+
+  src = get_source_file (self, context, error);
+  if (src == NULL)
+    return FALSE;
+
+  skip = builder_source_dir_get_skip (source, context);
+  g_mkdir_with_parents (flatpak_file_get_path_cached (src), 0755);
+  if (!flatpak_cp_a (src, dest,
+                     FLATPAK_CP_FLAGS_MERGE|FLATPAK_CP_FLAGS_NO_CHOWN,
+                     skip, NULL, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+builder_source_dir_bundle (BuilderSource  *source,
+                            BuilderContext *context,
+                            GError        **error)
+{
+  BuilderSourceDir *self = BUILDER_SOURCE_DIR (source);
+  g_autoptr(GFile) src = NULL;
+  g_autoptr(GFile) dest = NULL;
+  GFile *manifest_base_dir = builder_context_get_base_dir (context);
+  g_autofree char *rel_path = NULL;
+  g_autoptr(GPtrArray) skip = NULL;
+
+  src = get_source_file (self, context, error);
+  if (src == NULL)
+    return FALSE;
+
+  rel_path = g_file_get_relative_path (manifest_base_dir, src);
+  if (rel_path == NULL)
+    {
+      g_warning ("Local file %s is outside manifest tree, not bundling", flatpak_file_get_path_cached (src));
+      return TRUE;
+    }
+
+  dest = flatpak_build_file (builder_context_get_app_dir (context),
+                             "sources/manifest", rel_path, NULL);
+
+  skip = builder_source_dir_get_skip (source, context);
+  g_mkdir_with_parents (flatpak_file_get_path_cached (dest), 0755);
+  if (!flatpak_cp_a (src, dest,
+                     FLATPAK_CP_FLAGS_MERGE|FLATPAK_CP_FLAGS_NO_CHOWN,
+                     skip, NULL, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+builder_source_dir_update (BuilderSource  *source,
+                            BuilderContext *context,
+                            GError        **error)
+{
+  return TRUE;
+}
+
+static void
+builder_source_dir_checksum (BuilderSource  *source,
+                              BuilderCache   *cache,
+                              BuilderContext *context)
+{
+  /* We can't realistically checksum a directory, so always rebuild */
+  builder_cache_checksum_random (cache);
+}
+
+static void
+builder_source_dir_class_init (BuilderSourceDirClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+  BuilderSourceClass *source_class = BUILDER_SOURCE_CLASS (klass);
+
+  object_class->finalize = builder_source_dir_finalize;
+  object_class->get_property = builder_source_dir_get_property;
+  object_class->set_property = builder_source_dir_set_property;
+
+  source_class->show_deps = builder_source_dir_show_deps;
+  source_class->download = builder_source_dir_download;
+  source_class->extract = builder_source_dir_extract;
+  source_class->bundle = builder_source_dir_bundle;
+  source_class->update = builder_source_dir_update;
+  source_class->checksum = builder_source_dir_checksum;
+
+  g_object_class_install_property (object_class,
+                                   PROP_PATH,
+                                   g_param_spec_string ("path",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_SKIP,
+                                   g_param_spec_boxed ("skip",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
+}
+
+static void
+builder_source_dir_init (BuilderSourceDir *self)
+{
+}

src/builder-source-dir.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright © 2015 Red Hat, Inc
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Alexander Larsson <alexl@redhat.com>
+ */
+
+#ifndef __BUILDER_SOURCE_DIR_H__
+#define __BUILDER_SOURCE_DIR_H__
+
+#include "builder-source.h"
+
+G_BEGIN_DECLS
+
+typedef struct BuilderSourceDir BuilderSourceDir;
+
+#define BUILDER_TYPE_SOURCE_DIR (builder_source_dir_get_type ())
+#define BUILDER_SOURCE_DIR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), BUILDER_TYPE_SOURCE_DIR, BuilderSourceDir))
+#define BUILDER_IS_SOURCE_DIR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), BUILDER_TYPE_SOURCE_DIR))
+
+GType builder_source_dir_get_type (void);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderSourceDir, g_object_unref)
+
+G_END_DECLS
+
+#endif /* __BUILDER_SOURCE_DIR_H__ */

src/builder-source-file.c

@@ -38,6 +38,7 @@ struct BuilderSourceFile
 
   char         *path;
   char         *url;
+  char        **mirror_urls;
   char         *md5;
   char         *sha1;
   char         *sha256;
@@ -61,6 +62,7 @@ enum {
   PROP_SHA256,
   PROP_SHA512,
   PROP_DEST_FILENAME,
+  PROP_MIRROR_URLS,
   LAST_PROP
 };
 
@@ -76,6 +78,7 @@ builder_source_file_finalize (GObject *object)
   g_free (self->sha256);
   g_free (self->sha512);
   g_free (self->dest_filename);
+  g_strfreev (self->mirror_urls);
 
   G_OBJECT_CLASS (builder_source_file_parent_class)->finalize (object);
 }
@@ -118,6 +121,10 @@ builder_source_file_get_property (GObject    *object,
       g_value_set_string (value, self->dest_filename);
       break;
 
+    case PROP_MIRROR_URLS:
+      g_value_set_boxed (value, self->mirror_urls);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -130,6 +137,7 @@ builder_source_file_set_property (GObject      *object,
                                   GParamSpec   *pspec)
 {
   BuilderSourceFile *self = BUILDER_SOURCE_FILE (object);
+  gchar **tmp;
 
   switch (prop_id)
     {
@@ -168,6 +176,12 @@ builder_source_file_set_property (GObject      *object,
       self->dest_filename = g_value_dup_string (value);
       break;
 
+    case PROP_MIRROR_URLS:
+      tmp = self->mirror_urls;
+      self->mirror_urls = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -276,9 +290,9 @@ get_source_file (BuilderSourceFile *self,
 }
 
 static GBytes *
-download_uri (const char     *url,
-              BuilderContext *context,
-              GError        **error)
+download_data_uri (const char     *url,
+                   BuilderContext *context,
+                   GError        **error)
 {
   SoupSession *session;
 
@@ -347,30 +361,22 @@ builder_source_file_download (BuilderSource  *source,
 
   if (g_file_query_exists (file, NULL))
     {
-      if (is_local && checksums[0] != NULL)
-        {
-          g_autofree char *data = NULL;
-          gsize len;
-
-          if (!g_file_load_contents (file, NULL, &data, &len, NULL, error))
-            return FALSE;
-
-          if (!builder_verify_checksums (base_name,
-                                         data, len,
-                                         checksums, checksums_type,
-                                         error))
-            return FALSE;
-        }
-      return TRUE;
+      return !is_local || checksums[0] == NULL ||
+             builder_verify_checksums (base_name, file,
+                                       checksums, checksums_type,
+                                       error);
     }
 
+  if (is_inline)
+    return TRUE;
+
   if (is_local)
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't find file at %s", self->path);
       return FALSE;
     }
 
-  if (checksums[0] == NULL && !is_inline)
+  if (checksums[0] == NULL)
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "No checksum specified for file source %s", base_name);
       return FALSE;
@@ -378,6 +384,7 @@ builder_source_file_download (BuilderSource  *source,
 
   if (!builder_context_download_uri (context,
                                      self->url,
+                                     (const char **)self->mirror_urls,
                                      file,
                                      checksums,
                                      checksums_type,
@@ -432,9 +439,9 @@ builder_source_file_extract (BuilderSource  *source,
     {
       g_autoptr(GBytes) content = NULL;
 
-      content = download_uri (self->url,
-                              context,
-                              error);
+      content = download_data_uri (self->url,
+                                   context,
+                                   error);
       if (content == NULL)
         return FALSE;
 
@@ -562,6 +569,7 @@ builder_source_file_checksum (BuilderSource  *source,
   builder_cache_checksum_compat_str (cache, self->sha1);
   builder_cache_checksum_compat_str (cache, self->sha512);
   builder_cache_checksum_str (cache, self->dest_filename);
+  builder_cache_checksum_compat_strv (cache, self->mirror_urls);
 }
 
 static void
@@ -630,6 +638,13 @@ builder_source_file_class_init (BuilderSourceFileClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_MIRROR_URLS,
+                                   g_param_spec_boxed ("mirror-urls",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
 }
 
 static void

src/builder-source-git.c

@@ -46,6 +46,7 @@ struct BuilderSourceGit
   char         *orig_ref;
   gboolean      disable_fsckobjects;
   gboolean      disable_shallow_clone;
+  gboolean      disable_submodules;
 };
 
 typedef struct
@@ -64,6 +65,7 @@ enum {
   PROP_COMMIT,
   PROP_DISABLE_FSCKOBJECTS,
   PROP_DISABLE_SHALLOW_CLONE,
+  PROP_DISABLE_SUBMODULES,
   LAST_PROP
 };
 
@@ -120,6 +122,10 @@ builder_source_git_get_property (GObject    *object,
       g_value_set_boolean (value, self->disable_shallow_clone);
       break;
 
+    case PROP_DISABLE_SUBMODULES:
+      g_value_set_boolean (value, self->disable_submodules);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -168,6 +174,10 @@ builder_source_git_set_property (GObject      *object,
       self->disable_shallow_clone = g_value_get_boolean (value);
       break;
 
+    case PROP_DISABLE_SUBMODULES:
+      self->disable_submodules = g_value_get_boolean (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -226,7 +236,7 @@ builder_source_git_download (BuilderSource  *source,
 {
   BuilderSourceGit *self = BUILDER_SOURCE_GIT (source);
   g_autofree char *location = NULL;
-  FlatpakGitMirrorFlags flags;
+  FlatpakGitMirrorFlags flags = 0;
 
   location = get_url_or_path (self, context, error);
   if (location == NULL)
@@ -235,7 +245,8 @@ builder_source_git_download (BuilderSource  *source,
   if (self->tag != NULL && self->branch != NULL)
     return flatpak_fail (error, "Both tag (%s) and branch (%s) specified for git source", self->tag, self->branch);
 
-  flags = FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
+  if (!self->disable_submodules)
+    flags |= FLATPAK_GIT_MIRROR_FLAGS_MIRROR_SUBMODULES;
   if (update_vcs)
     flags |= FLATPAK_GIT_MIRROR_FLAGS_UPDATE;
   if (self->disable_fsckobjects)
@@ -445,6 +456,14 @@ builder_source_git_class_init (BuilderSourceGitClass *klass)
                                                          "",
                                                          FALSE,
                                                          G_PARAM_READWRITE));
+
+  g_object_class_install_property (object_class,
+                                   PROP_DISABLE_SUBMODULES,
+                                   g_param_spec_boolean ("disable-submodules",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
 }
 
 static void

src/builder-source-patch.c

@@ -37,8 +37,10 @@ struct BuilderSourcePatch
   BuilderSource parent;
 
   char         *path;
+  char        **paths;
   guint         strip_components;
   gboolean      use_git;
+  gboolean      use_git_am;
   char        **options;
 };
 
@@ -52,9 +54,11 @@ G_DEFINE_TYPE (BuilderSourcePatch, builder_source_patch, BUILDER_TYPE_SOURCE);
 enum {
   PROP_0,
   PROP_PATH,
+  PROP_PATHS,
   PROP_STRIP_COMPONENTS,
   PROP_USE_GIT,
   PROP_OPTIONS,
+  PROP_USE_GIT_AM,
   LAST_PROP
 };
 
@@ -64,6 +68,7 @@ builder_source_patch_finalize (GObject *object)
   BuilderSourcePatch *self = (BuilderSourcePatch *) object;
 
   g_free (self->path);
+  g_strfreev (self->paths);
   g_strfreev (self->options);
 
   G_OBJECT_CLASS (builder_source_patch_parent_class)->finalize (object);
@@ -83,6 +88,10 @@ builder_source_patch_get_property (GObject    *object,
       g_value_set_string (value, self->path);
       break;
 
+    case PROP_PATHS:
+      g_value_set_boxed (value, self->paths);
+      break;
+
     case PROP_STRIP_COMPONENTS:
       g_value_set_uint (value, self->strip_components);
       break;
@@ -95,6 +104,10 @@ builder_source_patch_get_property (GObject    *object,
       g_value_set_boxed (value, self->options);
       break;
 
+    case PROP_USE_GIT_AM:
+      g_value_set_boolean (value, self->use_git_am);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
@@ -116,6 +129,12 @@ builder_source_patch_set_property (GObject      *object,
       self->path = g_value_dup_string (value);
       break;
 
+    case PROP_PATHS:
+      tmp = self->paths;
+      self->paths = g_strdupv (g_value_get_boxed (value));
+      g_strfreev (tmp);
+      break;
+
     case PROP_STRIP_COMPONENTS:
       self->strip_components = g_value_get_uint (value);
       break;
@@ -130,25 +149,40 @@ builder_source_patch_set_property (GObject      *object,
       g_strfreev (tmp);
       break;
 
+    case PROP_USE_GIT_AM:
+      self->use_git_am = g_value_get_boolean (value);
+      break;
+
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     }
 }
 
-static GFile *
-get_source_file (BuilderSourcePatch *self,
+static GPtrArray *
+get_source_files (BuilderSourcePatch *self,
                  BuilderContext     *context,
                  GError            **error)
 {
+  g_autoptr(GPtrArray) res = g_ptr_array_new_with_free_func (g_object_unref);
   GFile *base_dir = BUILDER_SOURCE (self)->base_dir;
+  int i;
 
-  if (self->path == NULL || self->path[0] == 0)
+  if ((self->path != NULL && self->path[0] != 0))
+    g_ptr_array_add (res, g_file_resolve_relative_path (base_dir, self->path));
+
+  if (self->paths != NULL)
+    {
+      for (i = 0; self->paths[i] != NULL; i++)
+        g_ptr_array_add (res, g_file_resolve_relative_path (base_dir, self->paths[i]));
+    }
+
+  if (res->len == 0)
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "path not specified");
       return NULL;
     }
 
-  return g_file_resolve_relative_path (base_dir, self->path);
+  return g_steal_pointer (&res);
 }
 
 static gboolean
@@ -156,10 +190,14 @@ builder_source_patch_show_deps (BuilderSource  *source,
                                 GError        **error)
 {
   BuilderSourcePatch *self = BUILDER_SOURCE_PATCH (source);
+  int i;
 
   if (self->path && self->path[0] != 0)
     g_print ("%s\n", self->path);
 
+  for (i = 0; self->paths != NULL && self->paths[i] != NULL; i++)
+    g_print ("%s\n", self->paths[i]);
+
   return TRUE;
 }
 
@@ -170,17 +208,21 @@ builder_source_patch_download (BuilderSource  *source,
                                GError        **error)
 {
   BuilderSourcePatch *self = BUILDER_SOURCE_PATCH (source);
+  g_autoptr(GPtrArray) srcs = NULL;
+  int i;
 
-  g_autoptr(GFile) src = NULL;
-
-  src = get_source_file (self, context, error);
-  if (src == NULL)
+  srcs = get_source_files (self, context, error);
+  if (srcs == NULL)
     return FALSE;
 
-  if (!g_file_query_exists (src, NULL))
+  for (i = 0; i < srcs->len; i++)
     {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't find file at %s", self->path);
-      return FALSE;
+      GFile *src = g_ptr_array_index (srcs, i);
+      if (!g_file_query_exists (src, NULL))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't find file at %s", self->path);
+          return FALSE;
+        }
     }
 
   return TRUE;
@@ -189,6 +231,7 @@ builder_source_patch_download (BuilderSource  *source,
 static gboolean
 patch (GFile      *dir,
        gboolean    use_git,
+       gboolean    use_git_am,
        const char *patch_path,
        char      **extra_options,
        GError    **error,
@@ -207,6 +250,10 @@ patch (GFile      *dir,
     g_ptr_array_add (args, "git");
     g_ptr_array_add (args, "apply");
     g_ptr_array_add (args, "-v");
+  } else if (use_git_am) {
+    g_ptr_array_add (args, "git");
+    g_ptr_array_add (args, "am");
+    g_ptr_array_add (args, "--keep-cr");
   } else {
     g_ptr_array_add (args, "patch");
   }
@@ -214,7 +261,7 @@ patch (GFile      *dir,
     g_ptr_array_add (args, (gchar *) extra_options[i]);
   while ((arg = va_arg (ap, const gchar *)))
     g_ptr_array_add (args, (gchar *) arg);
-  if (use_git) {
+  if (use_git || use_git_am) {
     g_ptr_array_add (args, (char *) patch_path);
   } else {
     g_ptr_array_add (args, "-i");
@@ -239,20 +286,34 @@ builder_source_patch_extract (BuilderSource  *source,
                               GError        **error)
 {
   BuilderSourcePatch *self = BUILDER_SOURCE_PATCH (source);
-
-  g_autoptr(GFile) patchfile = NULL;
-  g_autofree char *patch_path = NULL;
   g_autofree char *strip_components = NULL;
+  g_autoptr(GPtrArray) srcs = NULL;
+  int i;
 
-  patchfile = get_source_file (self, context, error);
-  if (patchfile == NULL)
+  if (self->use_git && self->use_git_am)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Patch '%s' cannot be applied: Both 'use-git' and 'use-git-am' are set. Only one can be set at a time",
+                   self->path);
+      return FALSE;
+    }
+
+  srcs = get_source_files (self, context, error);
+  if (srcs == NULL)
     return FALSE;
 
-  g_print ("Applying patch %s\n", self->path);
   strip_components = g_strdup_printf ("-p%u", self->strip_components);
-  patch_path = g_file_get_path (patchfile);
-  if (!patch (dest, self->use_git, patch_path, self->options, error, strip_components, NULL))
-    return FALSE;
+
+  for (i = 0; i < srcs->len; i++)
+    {
+      GFile *patchfile = g_ptr_array_index (srcs, i);
+      g_autofree char *basename = g_file_get_basename (patchfile);
+      g_autofree char *patch_path = g_file_get_path (patchfile);
+
+      g_print ("Applying patch %s\n", basename);
+      if (!patch (dest, self->use_git, self->use_git_am, patch_path, self->options, error, strip_components, NULL))
+        return FALSE;
+    }
 
   return TRUE;
 }
@@ -264,37 +325,42 @@ builder_source_patch_bundle (BuilderSource  *source,
 {
   BuilderSourcePatch *self = BUILDER_SOURCE_PATCH (source);
   GFile *manifest_base_dir = builder_context_get_base_dir (context);
-  g_autoptr(GFile) src = NULL;
-  g_autoptr(GFile) destination_file = NULL;
-  g_autofree char *rel_path = NULL;
-  g_autoptr(GFile) destination_dir = NULL;
+  g_autoptr(GPtrArray) srcs = NULL;
+  int i;
 
-  src = get_source_file (self, context, error);
-
-  if (src == NULL)
+  srcs = get_source_files (self, context, error);
+  if (srcs == NULL)
     return FALSE;
 
-  rel_path = g_file_get_relative_path (manifest_base_dir, src);
-  if (rel_path == NULL)
+  for (i = 0; i < srcs->len; i++)
     {
-      g_warning ("Patch %s is outside manifest tree, not bundling", flatpak_file_get_path_cached (src));
-      return TRUE;
+      GFile *src = g_ptr_array_index (srcs, i);
+      g_autofree char *rel_path = NULL;
+      g_autoptr(GFile) destination_file = NULL;
+      g_autoptr(GFile) destination_dir = NULL;
+
+      rel_path = g_file_get_relative_path (manifest_base_dir, src);
+      if (rel_path == NULL)
+        {
+          g_warning ("Patch %s is outside manifest tree, not bundling", flatpak_file_get_path_cached (src));
+          return TRUE;
+        }
+
+      destination_file = flatpak_build_file (builder_context_get_app_dir (context),
+                                             "sources/manifest", rel_path, NULL);
+
+      destination_dir = g_file_get_parent (destination_file);
+      if (!flatpak_mkdir_p (destination_dir, NULL, error))
+        return FALSE;
+
+      if (!g_file_copy (src, destination_file,
+                        G_FILE_COPY_OVERWRITE,
+                        NULL,
+                        NULL, NULL,
+                        error))
+        return FALSE;
     }
 
-  destination_file = flatpak_build_file (builder_context_get_app_dir (context),
-                                         "sources/manifest", rel_path, NULL);
-
-  destination_dir = g_file_get_parent (destination_file);
-  if (!flatpak_mkdir_p (destination_dir, NULL, error))
-    return FALSE;
-
-  if (!g_file_copy (src, destination_file,
-                    G_FILE_COPY_OVERWRITE,
-                    NULL,
-                    NULL, NULL,
-                    error))
-    return FALSE;
-
   return TRUE;
 }
 
@@ -304,19 +370,23 @@ builder_source_patch_checksum (BuilderSource  *source,
                                BuilderContext *context)
 {
   BuilderSourcePatch *self = BUILDER_SOURCE_PATCH (source);
+  g_autoptr(GPtrArray) srcs = NULL;
+  int i;
 
-  g_autoptr(GFile) src = NULL;
-  g_autofree char *data = NULL;
-  gsize len;
+  srcs = get_source_files (self, context, NULL);
 
-  src = get_source_file (self, context, NULL);
-  if (src == NULL)
-    return;
+  for (i = 0; srcs != NULL && i < srcs->len; i++)
+    {
+      GFile *src = g_ptr_array_index (srcs, i);
+      g_autofree char *data = NULL;
+      gsize len;
 
-  if (g_file_load_contents (src, NULL, &data, &len, NULL, NULL))
-    builder_cache_checksum_data (cache, (guchar *) data, len);
+      if (g_file_load_contents (src, NULL, &data, &len, NULL, NULL))
+        builder_cache_checksum_data (cache, (guchar *) data, len);
+    }
 
   builder_cache_checksum_str (cache, self->path);
+  builder_cache_checksum_compat_strv (cache, self->paths);
   builder_cache_checksum_uint32 (cache, self->strip_components);
   builder_cache_checksum_strv (cache, self->options);
 }
@@ -344,6 +414,13 @@ builder_source_patch_class_init (BuilderSourcePatchClass *klass)
                                                         "",
                                                         NULL,
                                                         G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_PATHS,
+                                   g_param_spec_boxed ("paths",
+                                                       "",
+                                                       "",
+                                                       G_TYPE_STRV,
+                                                       G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_STRIP_COMPONENTS,
                                    g_param_spec_uint ("strip-components",
@@ -366,6 +443,13 @@ builder_source_patch_class_init (BuilderSourcePatchClass *klass)
                                                        "",
                                                        G_TYPE_STRV,
                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_USE_GIT_AM,
+                                   g_param_spec_boolean ("use-git-am",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
 }
 
 static void

src/builder-source-svn.c

@@ -0,0 +1,427 @@
+/* builder-source-svn.c
+ *
+ * Copyright (C) 2015 Red Hat, Inc
+ *
+ * This file is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Alexander Larsson <alexl@redhat.com>
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/statfs.h>
+
+#include "builder-utils.h"
+
+#include "builder-source-svn.h"
+#include "builder-utils.h"
+#include "builder-flatpak-utils.h"
+
+struct BuilderSourceSvn
+{
+  BuilderSource parent;
+
+  char         *url;
+  char         *revision;
+  char         *orig_revision;
+};
+
+typedef struct
+{
+  BuilderSourceClass parent_class;
+} BuilderSourceSvnClass;
+
+G_DEFINE_TYPE (BuilderSourceSvn, builder_source_svn, BUILDER_TYPE_SOURCE);
+
+enum {
+  PROP_0,
+  PROP_URL,
+  PROP_REVISION,
+  LAST_PROP
+};
+
+static gboolean
+svn (GFile   *dir,
+     char   **output,
+     GError **error,
+     ...)
+{
+  gboolean res;
+  va_list ap;
+
+  va_start (ap, error);
+  res = flatpak_spawn (dir, output, 0, error, "svn", ap);
+  va_end (ap);
+
+  return res;
+}
+
+static gboolean
+cp (GError **error,
+     ...)
+{
+  gboolean res;
+  va_list ap;
+
+  va_start (ap, error);
+  res = flatpak_spawn (NULL, NULL, 0, error, "cp", ap);
+  va_end (ap);
+
+  return res;
+}
+
+static gboolean
+cp_dir (GFile *src_dir, GFile *dst_dir, GError **error)
+{
+  g_autofree char *src_path = g_strconcat (flatpak_file_get_path_cached (src_dir), "/", NULL);
+  g_autofree char *dst_path = g_strconcat (flatpak_file_get_path_cached (dst_dir), "/", NULL);
+
+  if (!cp (error, "-aT", src_path, dst_path, NULL))
+    return FALSE;
+
+  return TRUE;
+}
+
+static void
+builder_source_svn_finalize (GObject *object)
+{
+  BuilderSourceSvn *self = (BuilderSourceSvn *) object;
+
+  g_free (self->url);
+  g_free (self->revision);
+  g_free (self->orig_revision);
+
+  G_OBJECT_CLASS (builder_source_svn_parent_class)->finalize (object);
+}
+
+static void
+builder_source_svn_get_property (GObject    *object,
+                                 guint       prop_id,
+                                 GValue     *value,
+                                 GParamSpec *pspec)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (object);
+
+  switch (prop_id)
+    {
+    case PROP_URL:
+      g_value_set_string (value, self->url);
+      break;
+
+    case PROP_REVISION:
+      g_value_set_string (value, self->revision);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+builder_source_svn_set_property (GObject      *object,
+                                 guint         prop_id,
+                                 const GValue *value,
+                                 GParamSpec   *pspec)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (object);
+
+  switch (prop_id)
+    {
+    case PROP_URL:
+      g_free (self->url);
+      self->url = g_value_dup_string (value);
+      break;
+
+    case PROP_REVISION:
+      g_free (self->revision);
+      self->revision = g_value_dup_string (value);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static GFile *
+get_mirror_dir (BuilderSourceSvn *self, BuilderContext *context, const char *revision)
+{
+  g_autoptr(GFile) svn_dir = NULL;
+  g_autofree char *uri_filename = NULL;
+  g_autofree char *filename = NULL;
+  g_autofree char *svn_dir_path = NULL;
+
+  svn_dir = g_file_get_child (builder_context_get_state_dir (context),
+                              "svn");
+
+  svn_dir_path = g_file_get_path (svn_dir);
+  g_mkdir_with_parents (svn_dir_path, 0755);
+
+  uri_filename = builder_uri_to_filename (self->url);
+  if (revision)
+    filename = g_strconcat (uri_filename, "__r", revision, NULL);
+  else
+    filename = g_strdup (uri_filename);
+
+  return g_file_get_child (svn_dir, filename);
+}
+
+static char *
+get_current_revision (BuilderSourceSvn *self, BuilderContext *context, GError **error)
+{
+  g_autoptr(GFile) mirror_dir = NULL;
+  char *output = NULL;
+
+  mirror_dir = get_mirror_dir (self, context, self->revision);
+
+  if (!svn (mirror_dir, &output, error,
+            "info", "--non-interactive", "--show-item", "revision", NULL))
+    return NULL;
+
+  /* Trim trailing whitespace */
+  g_strchomp (output);
+
+  return output;
+}
+
+static gboolean
+builder_source_svn_download (BuilderSource  *source,
+                             gboolean        update_vcs,
+                             BuilderContext *context,
+                             GError        **error)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (source);
+  g_autoptr(GFile) parent = NULL;
+  g_autofree char *filename = NULL;
+
+  g_autoptr(GFile) mirror_dir = NULL;
+
+  if (self->url == NULL)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "URL not specified");
+      return FALSE;
+    }
+
+  mirror_dir = get_mirror_dir (self, context, self->revision);
+  parent = g_file_get_parent (mirror_dir);
+  filename = g_file_get_basename (mirror_dir);
+
+  if (!g_file_query_exists (mirror_dir, NULL))
+    {
+      g_autofree char *mirror_path = g_file_get_path (mirror_dir);
+      g_autofree char *path_tmp = g_strconcat (mirror_path, ".clone_XXXXXX", NULL);
+      g_autofree char *filename_tmp = NULL;
+      g_autoptr(GFile) mirror_dir_tmp = NULL;
+      g_autoptr(GFile) cached_svn_dir = NULL;
+
+      if (g_mkdtemp_full (path_tmp, 0755) == NULL)
+        return flatpak_fail (error, "Can't create temporary directory");
+
+      mirror_dir_tmp = g_file_new_for_path (path_tmp);
+      filename_tmp = g_file_get_basename (mirror_dir_tmp);
+
+      cached_svn_dir = builder_context_find_in_sources_dirs (context, "svn", filename, NULL);
+      if (cached_svn_dir != NULL)
+        {
+          if (!cp_dir (cached_svn_dir, mirror_dir_tmp, error))
+            return FALSE;
+
+          if (update_vcs)
+            {
+              g_print ("Updating svn repo %s\n", self->url);
+
+              if (!svn (parent, NULL, error,
+                        "update", "--non-interactive",
+                        "-r", self->revision ? self->revision : "HEAD",
+                        filename_tmp, NULL))
+                return FALSE;
+            }
+        }
+      else
+        {
+          g_print ("Getting svn repo %s\n", self->url);
+
+          if (!svn (parent, NULL, error,
+                    "checkout", "--non-interactive",
+                    "-r", self->revision ? self->revision : "HEAD",
+                    self->url,  filename_tmp, NULL))
+            return FALSE;
+        }
+
+
+      if (!g_file_move (mirror_dir_tmp, mirror_dir, 0, NULL, NULL, NULL, error))
+        return FALSE;
+    }
+  else if (update_vcs)
+    {
+      g_print ("Updating svn repo %s\n", self->url);
+
+      if (!svn (parent, NULL, error,
+                "update", "--non-interactive",
+                "-r", self->revision ? self->revision : "HEAD",
+                filename, NULL))
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+builder_source_svn_extract (BuilderSource  *source,
+                            GFile          *dest,
+                            BuilderOptions *build_options,
+                            BuilderContext *context,
+                            GError        **error)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (source);
+  g_autoptr(GFile) mirror_dir = NULL;
+  g_autofree char *dest_path = NULL;
+
+  mirror_dir = get_mirror_dir (self, context, self->revision);
+
+  dest_path = g_file_get_path (dest);
+  g_mkdir_with_parents (dest_path, 0755);
+
+  if (!cp_dir (mirror_dir, dest, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+builder_source_svn_bundle (BuilderSource  *source,
+                           BuilderContext *context,
+                           GError        **error)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (source);
+  g_autoptr(GFile) mirror_dir = NULL;
+  g_autoptr(GFile) dest_dir = NULL;
+  g_autofree char *base_name = NULL;
+
+  mirror_dir = get_mirror_dir (self, context, self->orig_revision);
+
+  if (mirror_dir == NULL)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't locate repo with URL '%s'", self->url);
+      return FALSE;
+    }
+
+  base_name = g_file_get_basename (mirror_dir);
+
+  dest_dir = flatpak_build_file (builder_context_get_app_dir (context),
+                                 "sources/svn", base_name,
+                                 NULL);
+
+  if (!flatpak_mkdir_p (dest_dir, NULL, error))
+    return FALSE;
+
+  if (!cp_dir (mirror_dir, dest_dir, error))
+    return FALSE;
+
+  if (self->orig_revision == NULL)
+    {
+      g_autoptr(GFile) alt_mirror_dir = get_mirror_dir (self, context, self->revision);
+      g_autofree char *alt_base_name = g_file_get_basename (alt_mirror_dir);
+      g_autoptr(GFile) alt_dest_dir = flatpak_build_file (builder_context_get_app_dir (context),
+                                                          "sources/svn", alt_base_name,
+                                                          NULL);
+      g_print ("alt_mirror_dir: %s (basename %s)\n", g_file_get_path (alt_mirror_dir), base_name);
+      if (!g_file_query_exists (alt_dest_dir, NULL) &&
+          !g_file_make_symbolic_link (alt_dest_dir, base_name, NULL, error))
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+static void
+builder_source_svn_checksum (BuilderSource  *source,
+                             BuilderCache   *cache,
+                             BuilderContext *context)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (source);
+  g_autofree char *current_revision = NULL;
+
+  g_autoptr(GError) error = NULL;
+
+  builder_cache_checksum_str (cache, self->url);
+  builder_cache_checksum_str (cache, self->revision);
+
+  current_revision = get_current_revision (self, context, &error);
+  if (current_revision)
+    builder_cache_checksum_str (cache, current_revision);
+  else if (error)
+    g_warning ("Failed to get current svn revision: %s", error->message);
+}
+
+static gboolean
+builder_source_svn_update (BuilderSource  *source,
+                           BuilderContext *context,
+                           GError        **error)
+{
+  BuilderSourceSvn *self = BUILDER_SOURCE_SVN (source);
+  char *current_revision;
+
+  self->orig_revision = g_strdup (self->revision);
+
+  current_revision = get_current_revision (self, context, NULL);
+  if (current_revision)
+    {
+      g_free (self->revision);
+      self->revision = current_revision;
+    }
+
+  return TRUE;
+}
+
+static void
+builder_source_svn_class_init (BuilderSourceSvnClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+  BuilderSourceClass *source_class = BUILDER_SOURCE_CLASS (klass);
+
+  object_class->finalize = builder_source_svn_finalize;
+  object_class->get_property = builder_source_svn_get_property;
+  object_class->set_property = builder_source_svn_set_property;
+
+  source_class->download = builder_source_svn_download;
+  source_class->extract = builder_source_svn_extract;
+  source_class->bundle = builder_source_svn_bundle;
+  source_class->update = builder_source_svn_update;
+  source_class->checksum = builder_source_svn_checksum;
+
+  g_object_class_install_property (object_class,
+                                   PROP_URL,
+                                   g_param_spec_string ("url",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_REVISION,
+                                   g_param_spec_string ("revision",
+                                                        "",
+                                                        "",
+                                                        NULL,
+                                                        G_PARAM_READWRITE));
+}
+
+static void
+builder_source_svn_init (BuilderSourceSvn *self)
+{
+}

src/builder-source-svn.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright © 2015 Red Hat, Inc
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *       Alexander Larsson <alexl@redhat.com>
+ */
+
+#ifndef __BUILDER_SOURCE_SVN_H__
+#define __BUILDER_SOURCE_SVN_H__
+
+#include "builder-source.h"
+
+G_BEGIN_DECLS
+
+typedef struct BuilderSourceSvn BuilderSourceSvn;
+
+#define BUILDER_TYPE_SOURCE_SVN (builder_source_svn_get_type ())
+#define BUILDER_SOURCE_SVN(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), BUILDER_TYPE_SOURCE_SVN, BuilderSourceSvn))
+#define BUILDER_IS_SOURCE_SVN(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), BUILDER_TYPE_SOURCE_SVN))
+
+GType builder_source_svn_get_type (void);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (BuilderSourceSvn, g_object_unref)
+
+G_END_DECLS
+
+#endif /* __BUILDER_SOURCE_SVN_H__ */

src/builder-source.c

@@ -33,7 +33,9 @@
 #include "builder-source-patch.h"
 #include "builder-source-git.h"
 #include "builder-source-bzr.h"
+#include "builder-source-svn.h"
 #include "builder-source-file.h"
+#include "builder-source-dir.h"
 #include "builder-source-script.h"
 #include "builder-source-shell.h"
 #include "builder-source-extra-data.h"
@@ -226,13 +228,18 @@ builder_source_find_property (JsonSerializable *serializable,
 {
   if (strcmp (name, "type") == 0)
     return NULL;
-  return builder_serializable_find_property_with_error (serializable, name);
+  return builder_serializable_find_property (serializable, name);
 }
 
 static void
 serializable_iface_init (JsonSerializableIface *serializable_iface)
 {
+  serializable_iface->serialize_property = builder_serializable_serialize_property;
+  serializable_iface->deserialize_property = builder_serializable_deserialize_property;
   serializable_iface->find_property = builder_source_find_property;
+  serializable_iface->list_properties = builder_serializable_list_properties;
+  serializable_iface->set_property = builder_serializable_set_property;
+  serializable_iface->get_property = builder_serializable_get_property;
 }
 
 JsonNode *
@@ -249,6 +256,8 @@ builder_source_to_json (BuilderSource *self)
     type = "archive";
   else if (BUILDER_IS_SOURCE_FILE (self))
     type = "file";
+  else if (BUILDER_IS_SOURCE_DIR (self))
+    type = "dir";
   else if (BUILDER_IS_SOURCE_SCRIPT (self))
     type = "script";
   else if (BUILDER_IS_SOURCE_SHELL (self))
@@ -261,6 +270,8 @@ builder_source_to_json (BuilderSource *self)
     type = "git";
   else if (BUILDER_IS_SOURCE_BZR (self))
     type = "bzr";
+  else if (BUILDER_IS_SOURCE_SVN (self))
+    type = "svn";
   else
     g_warning ("Unknown source type");
 
@@ -284,6 +295,8 @@ builder_source_from_json (JsonNode *node)
     return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_ARCHIVE, node);
   else if (strcmp (type, "file") == 0)
     return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_FILE, node);
+  else if (strcmp (type, "dir") == 0)
+    return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_DIR, node);
   else if (strcmp (type, "script") == 0)
     return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_SCRIPT, node);
   else if (strcmp (type, "shell") == 0)
@@ -296,6 +309,8 @@ builder_source_from_json (JsonNode *node)
     return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_GIT, node);
   else if (strcmp (type, "bzr") == 0)
     return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_BZR, node);
+  else if (strcmp (type, "svn") == 0)
+    return (BuilderSource *) json_gobject_deserialize (BUILDER_TYPE_SOURCE_SVN, node);
   else
     g_warning ("Unknown source type %s", type);
 

src/builder-utils.h

@@ -24,6 +24,7 @@
 #include <gio/gio.h>
 #include <libsoup/soup.h>
 #include <json-glib/json-glib.h>
+#include <curl/curl.h>
 
 #include <libxml/tree.h>
 
@@ -61,6 +62,21 @@ void     flatpak_collect_matches_for_path_pattern (const char *path,
 gboolean builder_migrate_locale_dirs (GFile   *root_dir,
                                       GError **error);
 
+GQuark builder_curl_error_quark (void);
+#define BUILDER_CURL_ERROR (builder_curl_error_quark ())
+
+GQuark builder_yaml_parse_error_quark (void);
+#define BUILDER_YAML_PARSE_ERROR (builder_yaml_parse_error_quark ())
+
+JsonNode * builder_json_node_from_data (const char *relpath,
+                                        const char *contents,
+                                        GError    **error);
+
+GObject * builder_gobject_from_data (GType       gtype,
+                                     const char *relpath,
+                                     const char *contents,
+                                     GError    **error);
+
 gboolean builder_host_spawnv (GFile                *dir,
                               char                **output,
                               GSubprocessFlags      flags,
@@ -76,7 +92,7 @@ gboolean builder_download_uri (SoupURI        *uri,
                                GFile          *dest,
                                const char     *checksums[BUILDER_CHECKSUMS_LEN],
                                GChecksumType   checksums_type[BUILDER_CHECKSUMS_LEN],
-                               SoupSession    *soup_session,
+                               CURL           *curl_session,
                                GError        **error);
 
 gsize builder_get_all_checksums (const char *checksums[BUILDER_CHECKSUMS_LEN],
@@ -87,14 +103,30 @@ gsize builder_get_all_checksums (const char *checksums[BUILDER_CHECKSUMS_LEN],
                                  const char *sha512);
 
 gboolean builder_verify_checksums (const char *name,
-                                   const char *data,
-                                   gsize len,
+                                   GFile *file,
                                    const char *checksums[BUILDER_CHECKSUMS_LEN],
                                    GChecksumType checksums_type[BUILDER_CHECKSUMS_LEN],
                                    GError **error);
 
-GParamSpec * builder_serializable_find_property_with_error (JsonSerializable *serializable,
-                                                            const char       *name);
+GParamSpec * builder_serializable_find_property (JsonSerializable *serializable,
+                                                 const char       *name);
+GParamSpec ** builder_serializable_list_properties (JsonSerializable *serializable,
+                                                    guint            *n_pspecs);
+gboolean builder_serializable_deserialize_property (JsonSerializable *serializable,
+                                                    const gchar      *property_name,
+                                                    GValue           *value,
+                                                    GParamSpec       *pspec,
+                                                    JsonNode         *property_node);
+JsonNode * builder_serializable_serialize_property (JsonSerializable *serializable,
+                                                    const gchar      *property_name,
+                                                    const GValue     *value,
+                                                    GParamSpec       *pspec);
+void builder_serializable_get_property (JsonSerializable *serializable,
+                                        GParamSpec       *pspec,
+                                        GValue           *value);
+void builder_serializable_set_property (JsonSerializable *serializable,
+                                        GParamSpec       *pspec,
+                                        const GValue     *value);
 
 void builder_set_term_title (const gchar *format,
                              ...) G_GNUC_PRINTF (1, 2);
@@ -111,6 +143,50 @@ xml_autoptr_cleanup_generic_free (void *p)
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (xmlDoc, xmlFreeDoc)
 
+typedef struct FlatpakXml FlatpakXml;
+
+struct FlatpakXml
+{
+  gchar      *element_name; /* NULL == text */
+  char      **attribute_names;
+  char      **attribute_values;
+  char       *text;
+  FlatpakXml *parent;
+  FlatpakXml *first_child;
+  FlatpakXml *last_child;
+  FlatpakXml *next_sibling;
+};
+
+FlatpakXml *flatpak_xml_new (const gchar *element_name);
+FlatpakXml *flatpak_xml_new_text (const gchar *text);
+void       flatpak_xml_add (FlatpakXml *parent,
+                            FlatpakXml *node);
+void       flatpak_xml_free (FlatpakXml *node);
+FlatpakXml *flatpak_xml_parse (GInputStream *in,
+                               gboolean      compressed,
+                               GCancellable *cancellable,
+                               GError      **error);
+void       flatpak_xml_to_string (FlatpakXml *node,
+                                  GString    *res);
+FlatpakXml *flatpak_xml_unlink (FlatpakXml *node,
+                                FlatpakXml *prev_sibling);
+FlatpakXml *flatpak_xml_find (FlatpakXml  *node,
+                              const char  *type,
+                              FlatpakXml **prev_child_out);
+
+GBytes *   flatpak_read_stream (GInputStream *in,
+                                gboolean      null_terminate,
+                                GError      **error);
+GVariant * flatpak_variant_compress (GVariant *variant);
+GVariant * flatpak_variant_uncompress (GVariant *variant, const GVariantType *type);
+
+
+gboolean flatpak_version_check (int major,
+                                int minor,
+                                int micro);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (FlatpakXml, flatpak_xml_free);
+
 G_END_DECLS
 
 #endif /* __BUILDER_UTILS_H__ */

tests/Makefile.am.inc

@@ -18,11 +18,14 @@ dist_installed_test_data = \
 	tests/libtest.sh \
 	tests/org.test.Hello.png \
 	tests/test.json \
+	tests/test.yaml \
 	tests/test-runtime.json \
 	tests/module1.json \
+	tests/module1.yaml \
 	tests/data1 \
 	tests/data1.patch \
 	tests/module2.json \
+	tests/module2.yaml \
 	tests/data2 \
 	tests/data2.patch \
 	tests/session.conf.in \

tests/libtest.sh

@@ -267,19 +267,19 @@ setup_python2_repo () {
 
 install_repo () {
     REPONAME=${1:-test}
-    ${FLATPAK} ${U} install ${REPONAME}-repo org.test.Platform master
-    ${FLATPAK} ${U} install ${REPONAME}-repo org.test.Hello master
+    ${FLATPAK} ${U} install -y ${REPONAME}-repo org.test.Platform master
+    ${FLATPAK} ${U} install -y ${REPONAME}-repo org.test.Hello master
 }
 
 install_sdk_repo () {
     REPONAME=${1:-test}
-    ${FLATPAK} ${U} install ${REPONAME}-repo org.test.Sdk master
+    ${FLATPAK} ${U} install -y ${REPONAME}-repo org.test.Sdk master
 }
 
 install_python2_repo () {
     REPONAME=${1:-test}
-    ${FLATPAK} ${U} install ${REPONAME}-repo org.test.PythonPlatform master
-    ${FLATPAK} ${U} install ${REPONAME}-repo org.test.PythonSdk master
+    ${FLATPAK} ${U} install -y ${REPONAME}-repo org.test.PythonPlatform master
+    ${FLATPAK} ${U} install -y ${REPONAME}-repo org.test.PythonSdk master
 }
 
 run () {

tests/make-test-runtime.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+# Don't inherit the -x from the testsuite
+set +x
+
 DIR=`mktemp -d`
 
 REPONAME=$1
@@ -18,7 +21,9 @@ cat > ${DIR}/metadata <<EOF
 name=${ID}
 EOF
 
-cat ${DIR}/metadata
+# On Debian derivatives, /usr/sbin and /sbin aren't in ordinary users'
+# PATHs, but ldconfig is kept in /sbin
+PATH="$PATH:/usr/sbin:/sbin"
 
 # Add bash and dependencies
 mkdir -p ${DIR}/usr/bin
@@ -30,11 +35,32 @@ if test -f /sbin/ldconfig.real; then
 else
     cp `which ldconfig` ${DIR}/usr/bin
 fi
-T=`mktemp`
+LIBS=`mktemp`
+BINS=`mktemp`
+
+add_bin() {
+    local f=$1
+    shift
+
+    if grep -qFe "${f}" $BINS; then
+        # Already handled
+        return 0
+    fi
+
+    echo $f >> $BINS
+
+    # Add library dependencies
+    (ldd "${f}" | sed "s/.* => //"  | awk '{ print $1}' | grep ^/ | sort -u -o $LIBS $LIBS -)  || true
+
+    local shebang=$(sed -n '1s/^#!\([^ ]*\).*/\1/p' "${f}")
+    if [ x$shebang != x ]; then
+        add_bin "$shebang"
+    fi
+}
+
 for i in $@; do
     I=`which $i`
-    cp $I ${DIR}/usr/bin
-    ldd $I | sed "s/.* => //"  | awk '{ print $1}' | grep ^/ | grep ^/ >> $T
+    add_bin $I
     if test $i == python2; then
         mkdir -p ${DIR}/usr/lib/python2.7/lib-dynload
         # This is a hardcoded minimal set of modules we need in the current tests.
@@ -61,10 +87,15 @@ for i in $@; do
         done
     fi
 done
-ln -s bash ${DIR}/usr/bin/sh
-for i in `sort -u $T`; do
+for i in `cat $BINS`; do
+    echo Adding binary $i 1>&2
+    cp "$i" ${DIR}/usr/bin/
+done
+for i in `cat $LIBS`; do
+    echo Adding library $i 1>&2
     cp "$i" ${DIR}/usr/lib/
 done
+ln -s bash ${DIR}/usr/bin/sh
 
 # We copy the C.UTF8 locale and call it en_US. Its a bit of a lie, but
 # the real en_US locale is often not available, because its in the

tests/module1.yaml

@@ -0,0 +1,14 @@
+name: module1
+modules:
+  - name: module1-first
+    buildsystem: simple
+    build-commands:
+      - 'echo module1 > /app/ran_module1'
+      - 'echo module1 > /app/modify_me'
+    sources:
+      - type: file
+        path: data1
+      - type: patch
+        strip-components: 0
+        path: data1.patch
+  - include2/module2.yaml

tests/module2.yaml

@@ -0,0 +1,11 @@
+name: module2
+buildsystem: simple
+ensure-writable: [ /modify_me ]
+build-commands:
+  - 'echo module2 > /app/ran_module2'
+  - 'echo module2 > /app/modify_me'
+sources:
+  - type: file
+    path: data2
+  - type: patch
+    path: data2.patch

tests/test-builder.sh

@@ -37,17 +37,21 @@ cd $TEST_DATA_DIR/
 cp -a $(dirname $0)/test-configure .
 echo "version1" > app-data
 cp $(dirname $0)/test.json .
+cp $(dirname $0)/test.yaml .
 cp $(dirname $0)/test-runtime.json .
 cp $(dirname $0)/0001-Add-test-logo.patch .
 mkdir include1
 cp $(dirname $0)/module1.json include1/
+cp $(dirname $0)/module1.yaml include1/
 cp $(dirname $0)/data1 include1/
 cp $(dirname $0)/data1.patch include1/
 mkdir include1/include2
 cp $(dirname $0)/module2.json include1/include2/
+cp $(dirname $0)/module2.yaml include1/include2/
 cp $(dirname $0)/data2 include1/include2/
 cp $(dirname $0)/data2.patch include1/include2/
 ${FLATPAK_BUILDER} --repo=$REPO $FL_GPGARGS --force-clean appdir test.json
+${FLATPAK_BUILDER} --repo=$REPO $FL_GPGARGS --force-clean appdir test.yaml
 
 assert_file_has_content appdir/files/share/app-data version1
 assert_file_has_content appdir/metadata shared=network;
@@ -69,7 +73,7 @@ assert_file_has_content hello_out2 '^Hello world2, from a sandbox$'
 
 echo "ok build"
 
-${FLATPAK} ${U} install test-repo org.test.Hello2 master
+${FLATPAK} ${U} install -y test-repo org.test.Hello2 master
 run org.test.Hello2 > hello_out3
 assert_file_has_content hello_out3 '^Hello world2, from a sandbox$'
 
@@ -81,8 +85,10 @@ echo "ok install+run"
 echo "version2" > app-data
 ${FLATPAK_BUILDER} $FL_GPGARGS --repo=$REPO --force-clean appdir test.json
 assert_file_has_content appdir/files/share/app-data version2
+${FLATPAK_BUILDER} $FL_GPGARGS --repo=$REPO --force-clean appdir test.yaml
+assert_file_has_content appdir/files/share/app-data version2
 
-${FLATPAK} ${U} update org.test.Hello2 master
+${FLATPAK} ${U} update -y org.test.Hello2 master
 
 run --command=cat org.test.Hello2 /app/share/app-data > app_data_2
 assert_file_has_content app_data_2 version2

tests/test.json

@@ -4,6 +4,7 @@
     "sdk": "org.test.Sdk",
     "command": "hello2.sh",
     "tags": ["test"],
+    "token-type": 0,
     "finish-args": [
         "--share=network"
     ],

tests/test.yaml

@@ -0,0 +1,53 @@
+app-id: org.test.Hello2
+runtime: org.test.Platform
+sdk: org.test.Sdk
+command: hello2.sh
+tags: [test]
+token-type: 0
+finish-args:
+  - --share=network
+build-options:
+  cflags: -O2 -g
+  cxxflags: -O2 -g
+  env:
+    FOO: bar
+    V: '1'
+cleanup: [/cleanup, '*.cleanup']
+cleanup-commands: [touch /app/cleaned_up]
+modules:
+  - include1/module1.yaml
+  - name: root
+    modules:
+      - name: test
+        config-opts: [--some-arg]
+        post-install:
+          - touch /app/bin/file.cleanup
+          - mkdir -p /app/share/icons/
+          - cp org.test.Hello.png /app/share/icons/
+        make-args: [BAR=2]
+        make-install-args: [BAR=3]
+        build-commands: ['echo foo > /app/out']
+        sources:
+          - type: file
+            path: test-configure
+            dest-filename: configure
+            sha256: 675a1ac2feec4d4f54e581b4b01bc3cfd2c1cf31aa5963574d31228c8a11b7e7
+          - type: file
+            path: app-data
+          - type: script
+            dest-filename: hello2.sh
+            commands: ['echo "Hello world2, from a sandbox"']
+          - type: shell
+            commands:
+              - mkdir /app/cleanup/
+              - touch /app/cleanup/a_file
+          - type: patch
+            path: 0001-Add-test-logo.patch
+            use-git: true
+      - name: test2
+        build-commands: ['echo foo2 > /app/out2']
+        buildsystem: simple
+        sources:
+          - type: file
+            path: app-data
+      - name: empty