Allow exclusion or selective inclusion of input files

.github/workflows/system-tests.yml

@@ -33,7 +33,7 @@ jobs:
             echo "Create package"
             cp $f tests/data/layout.yaml
             G_DEBUG=fatal-warnings partup -d -C tests/data package pkg.partup \
-              layout.yaml random.bin lorem.txt lorem.tar root.ext4
+              layout.yaml random.bin lorem.txt lorem.tar dir-struct.tar root.ext4
             echo "Show package content"
             sudo G_DEBUG=fatal-warnings partup -s show pkg.partup
             echo "Install package to loop device"

doc/layout-config-reference.rst

@@ -348,6 +348,30 @@ at least a ``filename``. For verifying the checksum of the given input file by
    checked against the provided file before writing to the target partition or
    volume.
 
+``exclude`` (sequence)
+   Paths to exclude for this partition. Paths to be excluded should be
+   provided as a sequence of strings. This only works on partitions with a valid
+   filesystem.
+
+   If the input is a ``.tar`` archive, the specified paths are excluded from
+   extraction using tar's ``--exclude`` option:
+   https://manpages.debian.org/unstable/tar/tar.1.en.html#exclude
+
+   For other input file types, the specified paths are deleted on the partition
+   after writing the input files.
+
+   ``exclude`` takes precedence over ``only``.
+
+``only`` (sequence)
+   A list of paths to only extract/keep on the corresponding partition. This
+   only works on partitions with a valid filesystem.
+
+   If the input is a ``.tar`` archive, only the specified members are extracted:
+   https://manpages.debian.org/unstable/tar/tar.1.en.html
+
+   For other input file types, any other paths are deleted on the partition
+   after writing the input files, except the ones specified.
+
 .. _supported-file-types:
 
 Supported File Types

src/pu-emmc.c

@@ -24,6 +24,8 @@ typedef struct _PuEmmcInput {
     gchar *filename;
     gchar *md5sum;
     gchar *sha256sum;
+    GList *exclude;
+    GList *only;
 
     /* Internal members */
     gsize _size;
@@ -375,7 +377,8 @@ pu_emmc_write_data(PuFlash *flash,
             if (g_regex_match_simple(".tar", path, G_REGEX_CASELESS, 0)) {
                 if (!pu_mount(part_path, part_mount, NULL, NULL, error))
                     return FALSE;
-                if (!pu_archive_extract(path, part_mount, error))
+                if (!pu_archive_extract(path, part_mount, input->exclude,
+                                        input->only, error))
                     return FALSE;
                 if (!pu_umount(part_mount, error))
                     return FALSE;
@@ -387,6 +390,15 @@ pu_emmc_write_data(PuFlash *flash,
                     return FALSE;
                 if (!pu_set_ext_label(part_path, part->label, error))
                     return FALSE;
+                if (input->exclude || input->only) {
+                    if (!pu_mount(part_path, part_mount, NULL, NULL, error))
+                        return FALSE;
+                    if (!pu_path_remove_exclude_only(part_mount, input->exclude,
+                                                     input->only, error))
+                        return FALSE;
+                    if (!pu_umount(part_mount, error))
+                        return FALSE;
+                }
             } else if (!part->filesystem) {
                 if (!pu_write_raw(path, part_path, self->device, 0, 0, 0, error))
                     return FALSE;
@@ -395,6 +407,11 @@ pu_emmc_write_data(PuFlash *flash,
                     return FALSE;
                 if (!pu_file_copy(path, part_mount, error))
                     return FALSE;
+                if (input->exclude || input->only) {
+                    if (!pu_path_remove_exclude_only(part_mount, input->exclude,
+                                                     input->only, error))
+                        return FALSE;
+                }
                 if (!pu_umount(part_mount, error))
                     return FALSE;
             }
@@ -601,6 +618,8 @@ pu_emmc_class_finalize(GObject *object)
             g_free(in->filename);
             g_free(in->md5sum);
             g_free(in->sha256sum);
+            g_list_free(g_steal_pointer(&in->exclude));
+            g_list_free(g_steal_pointer(&in->only));
             g_free(in);
         }
         g_list_free(g_steal_pointer(&part->input));
@@ -1058,6 +1077,30 @@ pu_emmc_parse_partitions(PuEmmc *emmc,
                 input->filename = pu_hash_table_lookup_string(iv->data.mapping, "filename", "");
                 input->md5sum = pu_hash_table_lookup_string(iv->data.mapping, "md5sum", "");
                 input->sha256sum = pu_hash_table_lookup_string(iv->data.mapping, "sha256sum", "");
+                GList *exclude_list = pu_hash_table_lookup_list(iv->data.mapping, "exclude", NULL);
+                if (exclude_list) {
+                    for (GList *e = exclude_list; e; e = e->next) {
+                        PuConfigValue *ev = e->data;
+                        if (ev->type != PU_CONFIG_VALUE_TYPE_STRING) {
+                            g_set_error(error, PU_ERROR, PU_ERROR_EMMC_PARSE,
+                                        "'exclude' does not contain a sequence of strings");
+                            return FALSE;
+                        }
+                        input->exclude = g_list_prepend(input->exclude, ev->data.string);
+                    }
+                }
+                GList *only_list = pu_hash_table_lookup_list(iv->data.mapping, "only", NULL);
+                if (only_list) {
+                    for (GList *o = only_list; o; o = o->next) {
+                        PuConfigValue *ov = o->data;
+                        if (ov->type != PU_CONFIG_VALUE_TYPE_STRING) {
+                            g_set_error(error, PU_ERROR, PU_ERROR_EMMC_PARSE,
+                                        "'only' does not contain a sequence of strings");
+                            return FALSE;
+                        }
+                        input->only = g_list_prepend(input->only, ov->data.string);
+                    }
+                }
                 part->input = g_list_prepend(part->input, input);
 
                 g_debug("Parsed partition input: filename=%s md5sum=%s sha256sum=%s",

src/pu-utils.c

@@ -9,6 +9,7 @@
 #include <gio/gio.h>
 #include <glib.h>
 #include <glib/gstdio.h>
+#include <glob.h>
 #include <stdio.h>
 #include <blkid.h>
 #include <sys/stat.h>
@@ -59,19 +60,38 @@ pu_spawn_command_line_sync(const gchar *command_line,
 gboolean
 pu_archive_extract(const gchar *filename,
                    const gchar *dest,
+                   GList *exclude,
+                   GList *only,
                    GError **error)
 {
-    g_autofree gchar *cmd = NULL;
+    g_autoptr(GString) cmd = NULL;
 
     g_return_val_if_fail(filename != NULL, FALSE);
     g_return_val_if_fail(dest != NULL, FALSE);
     g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
 
     g_debug("Extracting '%s' to '%s'", filename, dest);
 
-    cmd = g_strdup_printf("tar -xf %s -C %s", filename, dest);
+    cmd = g_string_new("tar");
 
-    if (!pu_spawn_command_line_sync(cmd, error)) {
+    /* TODO: trailing slashes may cause problems here? */
+    for (GList *e = exclude; e; e = e->next) {
+        gchar *es = e->data;
+        g_string_append_printf(cmd, " --exclude %s", es);
+    }
+
+    g_string_append_printf(cmd, " -C %s -xf %s", dest, filename);
+
+    for (GList *o = only; o; o = o->next) {
+        gchar *os = o->data;
+        if (g_regex_match_simple("[!^*?\\[\\]]", os, 0, 0)) {
+            g_string_append_printf(cmd, " --wildcards %s", os);
+        } else {
+            g_string_append_printf(cmd, " --no-wildcards %s", os);
+        }
+    }
+
+    if (!pu_spawn_command_line_sync(cmd->str, error)) {
         g_prefix_error(error, "Failed extracting '%s' to '%s': ", filename, dest);
         return FALSE;
     }
@@ -584,3 +604,185 @@ pu_str_pre_remove(gchar *string,
 
     return string;
 }
+
+gboolean
+pu_file_remove_recursive(GFile *file,
+                         GHashTable *skip,
+                         GError **error)
+{
+    g_autoptr(GFileEnumerator) dir_enum = NULL;
+    g_autoptr(GFileInfo) info = NULL;
+    gboolean skip_delete = FALSE;
+
+    g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
+
+    if (skip) {
+        g_autofree gchar *file_path = NULL;
+        GHashTableIter iter;
+        gpointer key;
+
+        file_path = g_file_get_path(file);
+        g_hash_table_iter_init(&iter, skip);
+        while (g_hash_table_iter_next(&iter, &key, NULL)) {
+            if (g_str_equal(key, file_path)) {
+                /* Skip deletion of 'file_path' (and possible children), because
+                 * it gets retained */
+                return TRUE;
+            }
+            if (g_str_has_prefix(key, file_path)) {
+                /* Skip deletion of 'file_path', because prefix 'key' gets
+                 * retained, but still evaluate possible children */
+                skip_delete = TRUE;
+                break;
+            }
+        }
+    }
+
+    dir_enum = g_file_enumerate_children(file, G_FILE_ATTRIBUTE_STANDARD_NAME,
+                                         G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                         NULL, NULL);
+    if (dir_enum) {
+        while ((info = g_file_enumerator_next_file(dir_enum, NULL, NULL)) != NULL) {
+            g_autoptr(GFile) child = NULL;
+            child = g_file_enumerator_get_child(dir_enum, info);
+            if (!pu_file_remove_recursive(child, skip, error)) {
+                if (error) {
+                    g_prefix_error(error, "Failed recursive file removal");
+                } else {
+                    g_set_error(error, PU_ERROR, PU_ERROR_FAILED,
+                                "Failed recursive file removal");
+                }
+                return FALSE;
+            }
+        }
+    }
+
+    if (skip_delete) {
+        return TRUE;
+    }
+
+    return g_file_delete(file, NULL, error);
+}
+
+static GHashTable *
+canonicalize_path_list(GList *paths,
+                       GError **error)
+{
+    g_autoptr(GHashTable) table = NULL;
+    glob_t gl;
+    gboolean first = TRUE;
+
+    g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
+
+    if (!paths) {
+        return NULL;
+    }
+
+    table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+
+    /* TODO: support wildcard paths, like "*foo.txt", which should match any
+     * directory containing foo.txt. */
+    for (GList *p = paths; p; p = p->next) {
+        const gchar *ps = p->data;
+        gint flags = GLOB_NOSORT | (first ? 0 : GLOB_APPEND);
+        gint ret = glob(ps, flags, NULL, &gl);
+        g_debug("ps: %s", ps);
+
+        if (ret == 0) {
+            first = FALSE;
+        } else if (ret == GLOB_NOMATCH) {
+            g_debug("glob did not match anything, continue");
+            continue;
+        } else {
+            g_set_error(error, PU_ERROR, PU_ERROR_FAILED,
+                        "glob() failed on '%s': %d", ps, ret);
+            return NULL;
+        }
+    }
+    g_debug("gl_pathc: %ld", gl.gl_pathc);
+
+    if (!first) {
+        for (gsize i = 0; i < gl.gl_pathc; i++) {
+            gchar *canon = g_canonicalize_filename(gl.gl_pathv[i], NULL);
+            g_debug("canon: %s", canon);
+            g_hash_table_replace(table, canon, NULL);
+        }
+        globfree(&gl);
+    }
+
+    return g_steal_pointer(&table);
+}
+
+/* TODO: Reconsider function name: Should be remove recursive exclusion set */
+gboolean
+pu_path_remove_exclude_only(const gchar *path,
+                            GList *exclude,
+                            GList *only,
+                            GError **error)
+{
+    g_autoptr(GHashTable) exclude_table = NULL;
+    g_autoptr(GHashTable) only_table = NULL;
+    g_autoptr(GHashTable) all_table = NULL;
+    g_autofree GList *all = NULL;
+
+    g_return_val_if_fail(g_strcmp0(path, "") > 0, FALSE);
+    g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
+
+    if (exclude) {
+        /* Create a list of directories and files that match "exclude" */
+        g_debug("EXCLUDE");
+        exclude_table = canonicalize_path_list(exclude, error);
+        if (!exclude_table) {
+            g_prefix_error(error, "Failed parsing 'exclude' paths: ");
+            return FALSE;
+        }
+
+        /* Delete all entries in "exclude" */
+        GHashTableIter iter;
+        gpointer key;
+
+        g_hash_table_iter_init(&iter, exclude_table);
+        while (g_hash_table_iter_next(&iter, &key, NULL)) {
+            g_autoptr(GFile) file = NULL;
+            file = g_file_new_for_path(key);
+            if (!pu_file_remove_recursive(file, NULL, error)) {
+                return FALSE;
+            }
+        }
+    }
+
+    if (only) {
+        /* Create a list of directories and files that match "only" */
+        g_debug("ONLY");
+        only_table = canonicalize_path_list(only, error);
+        if (!only_table) {
+            g_prefix_error(error, "Failed parsing 'only' paths: ");
+            return FALSE;
+        }
+
+        /* Create a list of directories and files that matches everything */
+        g_debug("ALL");
+        all = g_list_prepend(all, g_build_filename(path, "*", NULL));
+        all_table = canonicalize_path_list(all, error);
+        if (!all_table) {
+            g_prefix_error(error, "Failed parsing 'all' paths: ");
+            return FALSE;
+        }
+
+        /* Delete everything, except entries in "only" */
+        g_debug("REMOVE");
+        GHashTableIter iter;
+        gpointer key;
+
+        g_hash_table_iter_init(&iter, all_table);
+        while (g_hash_table_iter_next(&iter, &key, NULL)) {
+            g_autoptr(GFile) file = NULL;
+            file = g_file_new_for_path(key);
+            if (!pu_file_remove_recursive(file, only_table, error)) {
+                return FALSE;
+            }
+        }
+    }
+
+    return TRUE;
+}

src/pu-utils.h

@@ -7,12 +7,15 @@
 #define PARTUP_UTILS_H
 
 #include <glib.h>
+#include <gio/gio.h>
 #include <parted/parted.h>
 
 gboolean pu_spawn_command_line_sync(const gchar *command_line,
                                     GError **error);
 gboolean pu_archive_extract(const gchar *filename,
                             const gchar *dest,
+                            GList *exclude,
+                            GList *only,
                             GError **error);
 gboolean pu_make_filesystem(const gchar *part,
                             const gchar *type,
@@ -65,5 +68,14 @@ gchar * pu_device_get_partition_pattern(const gchar *device,
                                         GError **error);
 gchar * pu_str_pre_remove(gchar *string,
                           guint n);
+/*GHashTable * pu_hash_table_intersect(GHashTable *set_a,
+                                     GHashTable *set_b);*/
+gboolean pu_file_remove_recursive(GFile *file,
+                                  GHashTable *skip,
+                                  GError **error);
+gboolean pu_path_remove_exclude_only(const gchar *path,
+                                     GList *exclude,
+                                     GList *only,
+                                     GError **error);
 
 #endif /* PARTUP_UTILS_H */