From 2adf3381d7d891d0afb8826b148a96f7536902bd Mon Sep 17 00:00:00 2001 From: Marko Semet Date: Sat, 11 Jan 2020 17:51:04 +0100 Subject: [PATCH] Add: Build with hash --- build.sh | 16 +++++ build_in_tmp.sh | 32 +++++++++ hash_modules.py | 185 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100755 build.sh create mode 100755 build_in_tmp.sh create mode 100755 hash_modules.py diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..c1025c5 --- /dev/null +++ b/build.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash +# Args: [] + +if [ -z "$BUILD_DIR" ]; then + BUILD_DIR=build +fi + +CONFIG_DIR="$(dirname "$0")" && +( + if [ -z "$4" ]; then + flatpak-builder $3 --arch "$2" --rebuild-on-sdk-change "$BUILD_DIR" "$1" + else + HASH="$("$CONFIG_DIR/hash_modules.py" "$1" "$2" | sed -n '1p')" && + flatpak-builder $3 --arch "$2" --rebuild-on-sdk-change --gpg-sign=winebarrels@marko10-000.de --repo "$4" -s "WB_HASH='${HASH}'" "$BUILD_DIR" "$1" + fi +) \ No newline at end of file diff --git a/build_in_tmp.sh b/build_in_tmp.sh new file mode 100755 index 0000000..6279c7a --- /dev/null +++ b/build_in_tmp.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +# Args: [] + +# Env checks +if [ "$(whoami)" != "root" ] +then + (>&2 echo Programm has to run as root.) + exit 1 +fi +if [ -z "$SUDO_USER" ] +then + (>&2 echo Programm has to be run as sudo. SUDO_USER not set.) + exit 1 +fi + +CONFIG_DIR="$(dirname "$0")" && +BUILD_DIR="$(sudo -u "$SUDO_USER" -H mktemp -d)" && +CONFIG_FILE="$(realpath "$1")" && +(>2 echo BUILD DIR: "$BUILD_DIR") && +sudo -u "$SUDO_USER" -H mkdir -p ".flatpak-builder/ccache" && +sudo -u "$SUDO_USER" -H mkdir -p ".flatpak-builder/downloads" && +sudo -u "$SUDO_USER" -H mkdir -p "$BUILD_DIR/.flatpak-builder/ccache" && +sudo -u "$SUDO_USER" -H ln -s "$(realpath ".flatpak-builder/downloads")" "$BUILD_DIR/.flatpak-builder/downloads" && +mount --bind ".flatpak-builder/ccache" "$BUILD_DIR/.flatpak-builder/ccache" && +cd "$BUILD_DIR" && +sudo -u "$SUDO_USER" -H "$CONFIG_DIR/build.sh" "$CONFIG_FILE" "$2" "$3" "$4" + +RESULT=$! +umount "$BUILD_DIR/.flatpak-builder/ccache" && +cd && +rm -rf "$BUILD_DIR" +exit $RESULT diff --git a/hash_modules.py b/hash_modules.py new file mode 100755 index 0000000..485d814 --- /dev/null +++ b/hash_modules.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import hashlib +import itertools +import json +import os +import re +import subprocess +import sys +import yaml + + +def _load_stdout(args): + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + result = p.stdout.read() + if p.returncode != 0: + raise RuntimeError("Process crashed with " + repr(p.returncode) + ". Args: " + repr(args)) + return result + + +def list_remotes(verbose=False): + result = _load_stdout(["flatpak", "remote-list", "--user", "--columns=name"]).decode().splitlines() + if verbose: + print("Remotes: " + ", ".join(result), file=sys.stderr) + return result + + +def commit_hashes(name, verbose=False): + result = [] + for i in list_remotes(verbose=verbose): + out = None + try: + out = _load_stdout(["flatpak", "remote-info", "--user", "-c", i, name]).lower() + except RuntimeError: + pass + if out is not None: + out = out.splitlines()[0] + for i in filter(lambda x: not(b"a"[0] <= x <= b"z"[0] or b"0"[0] <= x <= b"9"[0]), out): + raise ValueError("Wrong char: " + repr(bytes([i]))) + result.append(out.decode()) + if verbose: + print("Commit hashes for " + name + ": " + ", ".join(result), file=sys.stderr) + return result + + +__hash_find = re.compile(".*WB_HASH\\=\\'(?P[a-zA-Z0-9]*)\\'.*") +def wb_hashes(name, verbose=False): + result = [] + for i in list_remotes(verbose=verbose): + out = None + try: + out = _load_stdout(["flatpak", "remote-info", "--user", i, name]) + except RuntimeError: + pass + if out is not None: + if b"WB_HASH=" in out: + out = out[out.find(b"WB_HASH="):] + else: + continue + out = __hash_find.match(out.decode()) + if out is not None: + result.append(out.groupdict()["hash"].lower()) + else: + continue + if verbose: + print("Commit wb hashes for " + name + ": " + ", ".join(result), file=sys.stderr) + return result + + +class ModuleLoader(): + path = str + content = dict + __moduleFiles = {} + + def __module(self, modules): + for iID in range(len(modules)): + i = modules[iID] + if isinstance(i, str): + i = self.relative(i) + if i in ModuleLoader.__moduleFiles: + tmp = ModuleLoader.__moduleFiles[i] + else: + tmp = ModuleLoader(i) + modules[iID] = tmp.content + else: + if "modules" in i: + self.__module(i["modules"]) + + def __init__(self, source): + if not isinstance(source, str): + raise ValueError("source have to be string. It was a " + + repr(source) + ".") + self.path = os.path.abspath(source) + ModuleLoader.__moduleFiles[self.path] = self + self.content = yaml.load(open(self.path, "r").read(), + Loader=yaml.SafeLoader) + if "modules" in self.content: + self.__module(self.content["modules"]) + + def hash_content(self, function): + tmp = json.dumps(self.content, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, indent=None, + separators=("", ""), default=None, sort_keys=True) + tmp = tmp.encode("utf-8") + return function(tmp) + + def get_depends(self, arch): + version_src = "/" + arch + "/" + self.content["runtime-version"] + result = [self.content["sdk"] + version_src, + self.content["runtime"] + version_src] + return result + + def hashes_from_base_platforms(self, arch, function, verbose=False): + # Get source + source = None + def helper_func(content): + nonlocal source + source = content + self.hash_content(helper_func) + + # Build hashes + base_hashes = list(map(lambda x: list(map(lambda y: y.encode("utf-8"), commit_hashes(x, verbose=verbose))), self.get_depends(arch))) + result = list(map(lambda x: function(b"\x00".join(x) + b"\x00" + source), itertools.product(*base_hashes))) + if verbose: + print("Local hashes for: " + ", ".join(result), file=sys.stderr) + return result + + def old_wb_hashes(self, arch, verbose=False): + version_target = "/" + arch + "/" + self.content["branch"] + result = [] + result += wb_hashes(self.content["id"] + version_target, verbose=verbose) + if "id-platform" in self.content: + result += wb_hashes(self.content["id-platform"] + version_target, verbose=verbose) + return result + + def version_hashes(self, arch, verbose=False): + def func(data): + tmp = hashlib.sha3_256() + tmp.update(data) + return tmp.hexdigest() + return self.hashes_from_base_platforms(arch, func, verbose=verbose) + + def required_update(self, arch, verbose=False): + own_hash = set(self.version_hashes(arch, verbose=verbose)) + old_hashes = set(self.old_wb_hashes(arch, verbose=verbose)) + return len(own_hash.intersection(old_hashes)) == 0 + + def relative(self, file): + return os.path.abspath(os.path.join(os.path.split(self.path)[0], file)) + + +def get_need_update(source, arch, verbose=False): + mod = ModuleLoader(source) + return mod.required_update(arch, verbose=verbose) + +def get_own_hash(source, arch, verbose=False): + mod = ModuleLoader(source) + tmp = mod.version_hashes(arch, verbose=verbose) + if len(tmp) != 1: + raise ValueError("No unique version number possible.") + return tmp[0] + + +if __name__ == '__main__': + import argparse + + # Parse config + parser = argparse.ArgumentParser(description="Generate hash of a module.") + parser.add_argument("file", metavar="file", type=str, nargs=1, + help="File configuration to generate hash.") + parser.add_argument("arch", metavar="arch", type=str, nargs=1, + help="The arch to build.") + parser.add_argument("--require-build", dest="rebuild", action="store_const", + const=True, default=False, help="Returns 0 when outdated.") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_const", + const=True, default=False, help="Verbose information to stderr.") + args = parser.parse_args() + + if args.rebuild: + if get_need_update(args.file[0], args.arch[0], verbose=args.verbose): + exit(0) + else: + exit(1) + else: + print(get_own_hash(args.file[0], args.arch[0], verbose=args.verbose))