sysusers: allow force reusing existing user/group IDs (#8037)

On Debian/Ubuntu systems the default passwd/group files use a
slightly strange mapping. E.g. in passwd:
```
man:x:6:12::/var/cache/man:/sbin/nologin
```
and in group:
```
disk:x:6:
man:x:12:
```

This is not supported in systemd-sysusers right now because
sysusers will not re-use an existing uid/gid in its normal
mode of operation. Unfortunately this reuse is needed to
replicate the default Debian/Ubuntu users/groups.

This commit enforces reuse when the "uid:gid" syntax is used
to fix this.

I also added a test that replicates the Debian base-passwd
passwd/group file to ensure things are ok. 

16 files changed, 176 insertions, 29 deletions

diff --git a/src/sysusers/sysusers.c b/src/sysusers/sysusers.c
index e06b4b6d5..510d5fa59 100644
--- a/src/sysusers/sysusers.c
+++ b/src/sysusers/sysusers.c
@@ -64,7 +64,9 @@ typedef struct Item {
         uid_t uid;
 
         bool gid_set:1;
-        bool gid_must_exist:1;
+        // id_set_strict means that the group with the specified gid must
+        // exist and that the check if a uid clashes with a gid is skipped
+        bool id_set_strict:1;
         bool uid_set:1;
 
         bool todo_user:1;
@@ -801,7 +803,7 @@ static int write_files(void) {
         return 0;
 }
 
-static int uid_is_ok(uid_t uid, const char *name) {
+static int uid_is_ok(uid_t uid, const char *name, bool check_with_gid) {
         struct passwd *p;
         struct group *g;
         const char *n;
@@ -813,17 +815,21 @@ static int uid_is_ok(uid_t uid, const char *name) {
 
         /* Try to avoid using uids that are already used by a group
          * that doesn't have the same name as our new user. */
-        i = ordered_hashmap_get(todo_gids, GID_TO_PTR(uid));
-        if (i && !streq(i->name, name))
-                return 0;
+        if (check_with_gid) {
+                i = ordered_hashmap_get(todo_gids, GID_TO_PTR(uid));
+                if (i && !streq(i->name, name))
+                        return 0;
+        }
 
         /* Let's check the files directly */
         if (hashmap_contains(database_uid, UID_TO_PTR(uid)))
                 return 0;
 
-        n = hashmap_get(database_gid, GID_TO_PTR(uid));
-        if (n && !streq(n, name))
-                return 0;
+        if (check_with_gid) {
+                n = hashmap_get(database_gid, GID_TO_PTR(uid));
+                if (n && !streq(n, name))
+                        return 0;
+        }
 
         /* Let's also check via NSS, to avoid UID clashes over LDAP and such, just in case */
         if (!arg_root) {
@@ -834,13 +840,15 @@ static int uid_is_ok(uid_t uid, const char *name) {
                 if (!IN_SET(errno, 0, ENOENT))
                         return -errno;
 
-                errno = 0;
-                g = getgrgid((gid_t) uid);
-                if (g) {
-                        if (!streq(g->gr_name, name))
-                                return 0;
-                } else if (!IN_SET(errno, 0, ENOENT))
-                        return -errno;
+                if (check_with_gid) {
+                        errno = 0;
+                        g = getgrgid((gid_t) uid);
+                        if (g) {
+                                if (!streq(g->gr_name, name))
+                                        return 0;
+                        } else if (!IN_SET(errno, 0, ENOENT))
+                                return -errno;
+                }
         }
 
         return 1;
@@ -952,7 +960,7 @@ static int add_user(Item *i) {
 
         /* Try to use the suggested numeric uid */
         if (i->uid_set) {
-                r = uid_is_ok(i->uid, i->name);
+                r = uid_is_ok(i->uid, i->name, !i->id_set_strict);
                 if (r < 0)
                         return log_error_errno(r, "Failed to verify uid " UID_FMT ": %m", i->uid);
                 if (r == 0) {
@@ -970,7 +978,7 @@ static int add_user(Item *i) {
                         if (c <= 0 || !uid_range_contains(uid_range, n_uid_range, c))
                                 log_debug("User ID " UID_FMT " of file not suitable for %s.", c, i->name);
                         else {
-                                r = uid_is_ok(c, i->name);
+                                r = uid_is_ok(c, i->name, true);
                                 if (r < 0)
                                         return log_error_errno(r, "Failed to verify uid " UID_FMT ": %m", i->uid);
                                 else if (r > 0) {
@@ -984,7 +992,7 @@ static int add_user(Item *i) {
 
         /* Otherwise, try to reuse the group ID */
         if (!i->uid_set && i->gid_set) {
-                r = uid_is_ok((uid_t) i->gid, i->name);
+                r = uid_is_ok((uid_t) i->gid, i->name, true);
                 if (r < 0)
                         return log_error_errno(r, "Failed to verify uid " UID_FMT ": %m", i->uid);
                 if (r > 0) {
@@ -1002,7 +1010,7 @@ static int add_user(Item *i) {
                                 return r;
                         }
 
-                        r = uid_is_ok(search_uid, i->name);
+                        r = uid_is_ok(search_uid, i->name, true);
                         if (r < 0)
                                 return log_error_errno(r, "Failed to verify uid " UID_FMT ": %m", i->uid);
                         else if (r > 0)
@@ -1099,7 +1107,7 @@ static int add_group(Item *i) {
                 r = gid_is_ok(i->gid);
                 if (r < 0)
                         return log_error_errno(r, "Failed to verify gid " GID_FMT ": %m", i->gid);
-                if (i->gid_must_exist) {
+                if (i->id_set_strict) {
                         /* If we require the gid to already exist we can return here:
                          * r > 0: means the gid does not exist -> fail
                          * r == 0: means the gid exists -> nothing more to do.
@@ -1548,7 +1556,7 @@ static int parse_line(const char *fname, unsigned line, const char *buffer) {
                                         if (r < 0)
                                                 return log_error_errno(r, "Failed to parse GID: '%s': %m", id);
                                         i->gid_set = true;
-                                        i->gid_must_exist = true;
+                                        i->id_set_strict = true;
                                         free_and_replace(resolved_id, uid);
                                 }
                                 r = parse_uid(resolved_id, &i->uid);
@@ -1819,7 +1827,7 @@ int main(int argc, char *argv[]) {
         }
 
         if (!uid_range) {
-                /* Default to default range of 1..SYSTEMD_UID_MAX */
+                /* Default to default range of 1..SYSTEM_UID_MAX */
                 r = uid_range_add(&uid_range, &n_uid_range, 1, SYSTEM_UID_MAX);
                 if (r < 0) {
                         log_oom();
diff --git a/test/TEST-21-SYSUSERS/test-1.input b/test/TEST-21-SYSUSERS/test-1.input
index bffc2cd7e..297bbe350 100644
--- a/test/TEST-21-SYSUSERS/test-1.input
+++ b/test/TEST-21-SYSUSERS/test-1.input
@@ -1,3 +1,5 @@
+# Trivial smoke test that covers the most basic functionality
+#
 #Type Name ID  GECOS HOMEDIR
 u     u1   222 -     -
 g     g1   111 -     -
diff --git a/test/TEST-21-SYSUSERS/test-2.expected-group b/test/TEST-21-SYSUSERS/test-2.expected-group
index f98e85fcf..4d9042607 100644
--- a/test/TEST-21-SYSUSERS/test-2.expected-group
+++ b/test/TEST-21-SYSUSERS/test-2.expected-group
@@ -1 +1 @@
-u1:x:999:
+u1:x:SYSTEM_UID_MAX:
diff --git a/test/TEST-21-SYSUSERS/test-2.expected-passwd b/test/TEST-21-SYSUSERS/test-2.expected-passwd
index d907e483f..f438ed137 100644
--- a/test/TEST-21-SYSUSERS/test-2.expected-passwd
+++ b/test/TEST-21-SYSUSERS/test-2.expected-passwd
@@ -1 +1 @@
-u1:x:999:999:some gecos:/random/dir:/sbin/nologin
+u1:x:SYSTEM_UID_MAX:SYSTEM_UID_MAX:some gecos:/random/dir:/sbin/nologin
diff --git a/test/TEST-21-SYSUSERS/test-2.input b/test/TEST-21-SYSUSERS/test-2.input
index d8f31347a..bc3e18222 100644
--- a/test/TEST-21-SYSUSERS/test-2.input
+++ b/test/TEST-21-SYSUSERS/test-2.input
@@ -1,2 +1,4 @@
+# Trivial smoke test that generate the ID dynamically based on SYSTEM_UID_MAX
+#
 #Type Name ID  GECOS        HOMEDIR
 u     u1   -   "some gecos" /random/dir
diff --git a/test/TEST-21-SYSUSERS/test-3.input b/test/TEST-21-SYSUSERS/test-3.input
index b4f86a69f..3257082ce 100644
--- a/test/TEST-21-SYSUSERS/test-3.input
+++ b/test/TEST-21-SYSUSERS/test-3.input
@@ -1,3 +1,6 @@
+# Ensure that the semantic for the uid:gid syntax is correct
+#
+#Type Name ID  GECOS HOMEDIR
 g hoge    300     -            -
 u foo     301     -            -
 
diff --git a/test/TEST-21-SYSUSERS/test-4.input b/test/TEST-21-SYSUSERS/test-4.input
index 620423eab..557f61c42 100644
--- a/test/TEST-21-SYSUSERS/test-4.input
+++ b/test/TEST-21-SYSUSERS/test-4.input
@@ -1,3 +1,6 @@
+# Ensure that already created groups are used when using the uid:gid syntax
+#
+#Type Name ID  GECOS HOMEDIR
 g xxx 310
 u yyy 311:310
 u xxx 312:310
diff --git a/test/TEST-21-SYSUSERS/test-5.expected-group b/test/TEST-21-SYSUSERS/test-5.expected-group
new file mode 100644
index 000000000..e9ef0a799
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-5.expected-group
@@ -0,0 +1,39 @@
+adm:x:4:
+tty:x:5:
+disk:x:6:
+man:x:12:
+kmem:x:15:
+dialout:x:20:
+fax:x:21:
+voice:x:22:
+cdrom:x:24:
+floppy:x:25:
+tape:x:26:
+sudo:x:27:
+audio:x:29:
+dip:x:30:
+operator:x:37:
+src:x:40:
+shadow:x:42:
+utmp:x:43:
+video:x:44:
+sasl:x:45:
+plugdev:x:46:
+staff:x:50:
+games:x:60:
+users:x:100:
+nogroup:x:65534:
+root:x:0:
+daemon:x:1:
+bin:x:2:
+sys:x:3:
+lp:x:7:
+mail:x:8:
+news:x:9:
+uucp:x:10:
+proxy:x:13:
+www-data:x:33:
+backup:x:34:
+list:x:38:
+irc:x:39:
+gnats:x:41:
diff --git a/test/TEST-21-SYSUSERS/test-5.expected-passwd b/test/TEST-21-SYSUSERS/test-5.expected-passwd
new file mode 100644
index 000000000..116b12612
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-5.expected-passwd
@@ -0,0 +1,18 @@
+root:x:0:0::/root:/bin/sh
+daemon:x:1:1::/usr/sbin:/sbin/nologin
+bin:x:2:2::/bin:/sbin/nologin
+sys:x:3:3::/dev:/sbin/nologin
+sync:x:4:65534::/bin:/sbin/nologin
+games:x:5:60::/usr/games:/sbin/nologin
+man:x:6:12::/var/cache/man:/sbin/nologin
+lp:x:7:7::/var/spool/lpd:/sbin/nologin
+mail:x:8:8::/var/mail:/sbin/nologin
+news:x:9:9::/var/spool/news:/sbin/nologin
+uucp:x:10:10::/var/spool/uucp:/sbin/nologin
+proxy:x:13:13::/bin:/sbin/nologin
+www-data:x:33:33::/var/www:/sbin/nologin
+backup:x:34:34::/var/backups:/sbin/nologin
+list:x:38:38::/var/list:/sbin/nologin
+irc:x:39:39::/var/run/ircd:/sbin/nologin
+gnats:x:41:41::/var/lib/gnats:/sbin/nologin
+nobody:x:65534:65534::/nonexistent:/sbin/nologin
diff --git a/test/TEST-21-SYSUSERS/test-5.input b/test/TEST-21-SYSUSERS/test-5.input
new file mode 100644
index 000000000..57519d7c9
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-5.input
@@ -0,0 +1,47 @@
+# Reproduce the base-passwd master.{passwd,group} from Debian
+#
+#Type  Name        ID GECOS Home directory
+g      adm          4 -
+g      tty          5 -
+g      disk         6 -
+g      man         12 -
+g      kmem        15 -
+g      dialout     20 -
+g      fax         21 -
+g      voice       22 -
+g      cdrom       24 -
+g      floppy      25 -
+g      tape        26 -
+g      sudo        27 -
+g      audio       29 -
+g      dip         30 -
+g      operator    37 -
+g      src         40 -
+g      shadow      42 -
+g      utmp        43 -
+g      video       44 -
+g      sasl        45 -
+g      plugdev     46 -
+g      staff       50 -
+g      games       60 -
+g      users      100 -
+g      nogroup  65534 -
+
+u      root         0 -     /root
+u      daemon       1 -     /usr/sbin
+u      bin          2 -     /bin
+u      sys          3 -     /dev
+u      sync         4:65534 -     /bin
+u      games        5:60 -     /usr/games
+u      man          6:12 -     /var/cache/man
+u      lp           7 -     /var/spool/lpd
+u      mail         8 -     /var/mail
+u      news         9 -     /var/spool/news
+u      uucp        10 -     /var/spool/uucp
+u      proxy       13 -     /bin
+u      www-data    33 -     /var/www
+u      backup      34 -     /var/backups
+u      list        38 -     /var/list
+u      irc         39 -     /var/run/ircd
+u      gnats       41 -     /var/lib/gnats
+u      nobody   65534:65534 -     /nonexistent
diff --git a/test/TEST-21-SYSUSERS/test-6.expected-group b/test/TEST-21-SYSUSERS/test-6.expected-group
new file mode 100644
index 000000000..499c9008c
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-6.expected-group
@@ -0,0 +1,2 @@
+g1:x:111:
+u1:x:SYSTEM_UID_MAX:
diff --git a/test/TEST-21-SYSUSERS/test-6.expected-passwd b/test/TEST-21-SYSUSERS/test-6.expected-passwd
new file mode 100644
index 000000000..5af9d1142
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-6.expected-passwd
@@ -0,0 +1 @@
+u1:x:SYSTEM_UID_MAX:SYSTEM_UID_MAX::/:/sbin/nologin
diff --git a/test/TEST-21-SYSUSERS/test-6.input b/test/TEST-21-SYSUSERS/test-6.input
new file mode 100644
index 000000000..764f57e82
--- /dev/null
+++ b/test/TEST-21-SYSUSERS/test-6.input
@@ -0,0 +1,7 @@
+# Ensure that existing IDs are not reused by default. I.e. the existing
+# ID 111 from g1 will cause u1 to get a new and different ID (999 on most
+# systems).
+#
+#Type Name ID  GECOS HOMEDIR
+g     g1   111 -     -
+u     u1   111 -     -
diff --git a/test/TEST-21-SYSUSERS/test.sh b/test/TEST-21-SYSUSERS/test.sh
index 14f2b4ae0..f69d27748 100755
--- a/test/TEST-21-SYSUSERS/test.sh
+++ b/test/TEST-21-SYSUSERS/test.sh
@@ -10,6 +10,16 @@ test_setup() {
         mkdir -p $TESTDIR/etc  $TESTDIR/usr/lib/sysusers.d $TESTDIR/tmp
 }
 
+preprocess() {
+    in="$1"
+
+    # see meson.build how to extract this. gcc -E was used before to
+    # get this value from config.h, however the autopkgtest fails with
+    # it
+    SYSTEM_UID_MAX=$(awk 'BEGIN { uid=999 } /^\s*SYS_UID_MAX\s+/ { uid=$2 } END { print uid }' /etc/login.defs)
+    sed "s/SYSTEM_UID_MAX/${SYSTEM_UID_MAX}/g" "$in"
+}
+
 test_run() {
         # ensure our build of systemd-sysusers is run
         PATH=${BUILD_DIR}:$PATH
@@ -21,11 +31,11 @@ test_run() {
                 cp $f $TESTDIR/usr/lib/sysusers.d/test.conf
                 systemd-sysusers --root=$TESTDIR
 
-                if ! diff -u $TESTDIR/etc/passwd ${f%.*}.expected-passwd; then
+                if ! diff -u $TESTDIR/etc/passwd <(preprocess ${f%.*}.expected-passwd); then
                         echo "**** Unexpected output for $f"
                         exit 1
                 fi
-                if ! diff -u $TESTDIR/etc/group ${f%.*}.expected-group; then
+                if ! diff -u $TESTDIR/etc/group <(preprocess ${f%.*}.expected-group); then
                         echo "**** Unexpected output for $f"
                         exit 1
                 fi
diff --git a/test/TEST-21-SYSUSERS/unhappy-1.input b/test/TEST-21-SYSUSERS/unhappy-1.input
index 77390371d..b8ed85525 100644
--- a/test/TEST-21-SYSUSERS/unhappy-1.input
+++ b/test/TEST-21-SYSUSERS/unhappy-1.input
@@ -1 +1,4 @@
-u u1 9999999999 - -
\ No newline at end of file
+# Ensure invalid uids are detected
+#
+#Type Name ID  GECOS HOMEDIR
+u u1 9999999999 - -
diff --git a/test/TEST-21-SYSUSERS/unhappy-2.input b/test/TEST-21-SYSUSERS/unhappy-2.input
index 521c741cb..5be0e6d18 100644
--- a/test/TEST-21-SYSUSERS/unhappy-2.input
+++ b/test/TEST-21-SYSUSERS/unhappy-2.input
@@ -1,2 +1,4 @@
-# it is not allowed to create groups implicitely in the uid:gid syntax
-u u1 100:100 -
\ No newline at end of file
+# Ensure it is not allowed to create groups implicitely in the uid:gid syntax
+#
+#Type Name ID  GECOS HOMEDIR
+u u1 100:100 -