From d2ee698f9031cfd37eb45672d80bd51bceeee136 Mon Sep 17 00:00:00 2001 From: Rob Landley Date: Fri, 29 Oct 2021 22:41:38 -0500 Subject: [PATCH] More roadmap/mkstatus tweaks. Use more _cmd suffixes on anchor tags because and apparently stomp each other for links. Properly hide the shell builtins in the categories (there was code to do it but it had bit-rotted). Minor rewording of some roadmap descriptions. --- scripts/mkstatus.py | 9 +- www/roadmap.html | 201 ++++++++++++++++++++++---------------------- 2 files changed, 107 insertions(+), 103 deletions(-) diff --git a/scripts/mkstatus.py b/scripts/mkstatus.py index e49ab2ce..ea92a6f9 100755 --- a/scripts/mkstatus.py +++ b/scripts/mkstatus.py @@ -23,7 +23,7 @@ print "Collecting data..." stuff,blah=readit(["sed","-n", 's//\\1 /;t good;d;:good;h;:loop;n;s@@@;t out;H;b loop;:out;g;s/\\n/ /g;p', "www/roadmap.html", "www/status.html"]) blah,toystuff=readit(["./toybox"]) -blah,pending=readit(["sed -n 's/[^ \\t].*TOY(\\([^,]*\\),.*/\\1/p' toys/pending/*.c"], 1) +blah,pending=readit(["sed", "-n", "s/.*NEWTOY[(]\([^,]*\).*TOYFLAG_NOFORK.*/\1/p", "toys/pending/sh.c"], True) blah,version=readit(["git","describe","--tags"]) print "Analyzing..." @@ -53,8 +53,9 @@ conv = [("posix", '%%s', '(%s)'), ("toolbox", "", '{%s}'), ("klibc_cmd", "", '=%s='), ("sash_cmd", "", '#%s#'), ("sbase_cmd", "", '@%s@'), - ("beastiebox_cmd", "", '*%s*'), ("tizen", "", '$%s$'), - ("shell", "", "%%%s%%"), ("fhs_cmd", "", '-%s-'), ("yocto", "", ".%s."), + ("beastiebox_cmd", "", '*%s*'), ("tizen_cmd", "", '$%s$'), + ("fhs_cmd", "", '-%s-'), ("yocto_cmd", "", ".%s."), + ("shell", "", "%%%s%%"), ("request", '%%s', '+%s+')] @@ -98,7 +99,7 @@ print "implemented=%s" % len(toystuff) outfile=open("www/status.gen", "w") outfile.write("

Status of toybox %s

\n" % version[0]); -outfile.write("

Legend: %s pending

\n"%" ".join(map(lambda i: i[2]%(i[0].split("_")[0]), conv[:-1]))) +outfile.write("

Legend: %s pending

\n"%" ".join(map(lambda i: i[2]%(i[0].split("_")[0]), conv[:-2]))) outfile.write("

Completed

%s

\n" % "\n".join(done)) outfile.write("

Partially implemented (in toys/pending)

%s

\n" % "\n".join(pend)) diff --git a/www/roadmap.html b/www/roadmap.html index 400fa329..195de819 100644 --- a/www/roadmap.html +++ b/www/roadmap.html @@ -23,7 +23,7 @@ -

Introduction

+

Introduction (Goals and use cases)

We have several potential use cases for a new set of command line utilities, and are using those to determine which commands to implement @@ -31,19 +31,19 @@ for Toybox's 1.0 release. Most of these have their own section in the status page, showing current progress towards commplation.

-

The most interesting publicly available standards are POSIX-2008 (also -known as SUSv4), the Linux Standard Base (version 4.1), and the official -Linux man pages, all -of which include commands we've decided not implement and exclude -commands or features we have.

+

The most interesting publicly available standards are A) POSIX-2008 (also +known as SUSv4), B) the Linux Standard Base version 4.1, and C) the official +Linux man pages. +But they include commands we've decided not implement, exclude +commands or features we have, and don't always entirely match reality.

The most thorough real world test (other than a large interactive -userbase) is using toybox in a -development environment by providing the commands for a build system such as +userbase) is using toybox as the command line in a build system such as Aboriginal Linux, having it rebuild itself from source code, and using the result to build Linux From Scratch. -The current goal is to use mkroot +The current "minimal native development system" goal is to use +mkroot plus musl-cross-make to hermetically build AOSP.

@@ -98,7 +98,7 @@ to punch holes in the standard big enough to drive Windows NT and OS/360 through. When open source projects like Linux started developing on the internet -(accelerated by the 1993 relaxation of the National Science Foundation's +(enabled by the 1993 relaxation of the National Science Foundation's "Acceptable Use Policy" allowing everyone to connect to the internet, previously restricted to approved government/military/university organizations), Posix ignored @@ -120,6 +120,8 @@ themselves are missing dozens of features and specify silly things like ebcdic support in dd or that wc should use %d (not %lld) for byte counts. So we have to extensively filter posix to get a useful set of recommendations.

+

Analysis

+

Starting with the full "utilities" list, we first remove generally obsolete @@ -196,15 +198,18 @@ of those donors regardless of technical merit. (The Linux Foundation, which maintains the LSB, isn't a 501c3. It's a 501c6, the same kind of legal entity as the Tobacco Institute and Microsoft's -old "Don't Copy That Floppy" program.) Debian officially -washed its hands of LSB when 5.0 -came out in 2015, and no longer even pretends to support it (which may affect -Debian derivatives like Ubuntu and Knoppix). Toybox has stayed on 4.1 for -similar reasons.

+old "Don't Copy That Floppy" program.) Debian officially +washed its hands of LSB by +refusing to adopt release 5.0 in 2015, and no longer even pretends to support +it (which affect Debian derivatives like Ubuntu and Knoppix). Toybox has +stayed on 4.1 for similar reasons: a lot of historical effort went into +producing the standard before the Linux Foundation took over.

That said, Posix by itself isn't enough, and this is the next most comprehensive standards effort for Linux so far, so we salvage what we can.

+

Analysis

+

The LSB specifies a list of command line utilities:

@@ -246,12 +251,15 @@ su sync tar umount useradd userdel usermod zcat

IETF RFCs and Man Pages

-

They're very nice. There's thousands of them.

+

They're very nice, but there's thousands of them.

Discussion of standards wouldn't be complete without the Internet Engineering Task Force's "Request For Comments" collection and Michael Kerrisk's -Linux man-pages project, -but neither helps us select which commands to include.

+Linux man-pages project. +Except these aren't standards, they're collections of documentation with +low barriers to inclusion. They're not saying "you should support +X", they're saying "if you do, here's how". +Thus neither really helps us select which commands to include.

The man pages website includes the commands in git, yum, perf, postgres, flatpack... Great for examining the features of a command you've @@ -270,7 +278,7 @@ they document is to find an That said, RFC documents can be useful (especially for networking protocols) and the four URL templates the recommended starting files for new commands (toys/example/skeleton.c or toys/example/hello.c depending on how much -plumbing you want to start with) provide point to are posix, lsb, man, and +plumbing you want to start with) provide point to posix, lsb, man, and rfc pages.

@@ -520,7 +528,8 @@ with Intel's Moblin to form LiMo to form Tizen, -which became a Samsung-only project (that still ships inside televisions, +which became a Samsung-only project (that still ships +inside televisions, but was otherwise subsumed into Android GO).

Along the way, the Tizen project expressed a desire to eliminate GPLv3 software @@ -530,7 +539,7 @@ from its core system, and in installing toybox as

They had a fairly long list of new commands they wanted to see in toybox:

- + arch base64 users unexpand shred join csplit hostid nproc runcon sha224sum sha256sum sha384sum sha512sum sha3sum mkfs.vfat fsck.vfat dosfslabel uname pinky diff3 sdiff zcmp zdiff zegrep zfgrep zless zmore @@ -552,26 +561,26 @@ if/else setup. (We added lib/lsm.h to abstract this.)

Use case: Yocto

-

Another project the Linux Foundation got paid to appreciate is Yocto, +

Another project the Linux Foundation is paid to appreciate is Yocto, which was designed to fix the ongoing proprietary fragmentation problem -(now in Linux build systems instead of propreitary unixes) by being the +(now in Linux build systems instead of vendor unix forks) by being the build system equivalent of a glue trap. While proclaiming that having the "minimum level of standardization" contributes to a "strong ecosystem", Yocto uses a "layered" design where everybody who touches it is encouraged to add more and more layers of metadata on top of what came before, until they wind up using repo just to manage -the layers of metadata (let alone their contents). But (and this is the -important bit), all these dispirate forks are called yocto and built on -top of giant piles of code the Linux Foundation can take credit for. -(Although really it's a reskin of OpenEmbedded.)

+the layers (let alone their contents). But -- and this is the +important bit -- all these dispirate forks are called "yocto" and built on +top of giant piles of code the Linux Foundation can take credit for +since they filed the serial numbers off OpenEmbedded.

-

Yocto's "core-image-minimal" target (only 3,106 build stages in the 3.3 +

Yocto's "core-image-minimal" target (only 3,106 build steps in the 3.3 release, which believe it or not is an improvement) builds a busybox-based system with the following commands:

- -addgroup adduser ascii ash awk base32 basename blkid bunzip2 bzcat bzip2 cat + +addgroup adduser ascii sh awk base32 basename blkid bunzip2 bzcat bzip2 cat chattr chgrp chmod chown chroot chvt clear cmp cp cpio crc32 cut date dc dd deallocvt delgroup deluser depmod df diff dirname dmesg dnsdomainname du dumpkmap dumpleases echo egrep env expr false fbset fdisk fgrep find flock @@ -591,64 +600,63 @@ usleep vi watch wc wget which who whoami xargs xzcat yes zcat +
Filesystem Hierachy Standard

Filesystem Hierarchy Standard:

-

Another standard captured by the Linux Foundation. (At least the +

Another standard taken over by the Linux Foundation. (At least the links to this one didn't go 404 the instant they took it over). Of historical interest due to what it managed to achieve before they chased away the hobbyists maintaining it. Only one version (3.0 in 2015) has been released since the Linux Foundation -consumed the FHS. The previous release, Version 2.3, was released in 2004. +absorbed the FHS. The previous release, Version 2.3, was released in 2004. The Linux Foundation did not retain earlier versions. The contents of -the relevant sections appear identical between the two versions (the -Linux Foundation added section numbers, that appears to be it).

+the relevant sections appear identical between the two versions, the +Linux Foundation just added section numbers.

FHS 3.0 section 3.4.2 requires commands to be in the /bin directory, and then 3.4.3 has an optional list, -and then 3.16.2 and 3.16.3 similarly cover /sbin. (There are linux -specific sections in 6.1.2 and 6.1.6 but everything in them is obsolete.) -The contents appear to be identical between 2.3 and 3.0: the only change -the Linux Foundation made here was to add section numbers.

+and then 3.16.2 and 3.16.3 similarly cover /sbin. There are linux +specific sections in 6.1.2 and 6.1.6 but everything in them is obsolete.

-

The /bin options include csh but not bash, and ed but not vi, which -is crazy. The /sbin options has update which seems obsolete (filesystem +

The /bin options include csh but not bash, and ed but not vi. +The /sbin options have update which seems obsolete (filesystem buffers haven't needed a userspace process to flush them for DECADES), fastboot and fasthalt (reboot and halt have -nf), and -fsck.* and mkfs.* that don't actually specify specific filesystems. +fsck.* and mkfs.* that don't actually specify any specific filesystems. Removing that gives us:

cat chgrp chmod chown cp date dd df dmesg echo false hostname kill ln login ls mkdir mknod more mount mv ps pwd rm rmdir sed sh stty su sync true -umount uname -tar cpio gzip gunzip zcat netstat ping -shutdown -fdisk fsck getty halt ifconfig init mkfs mkswap reboot route swapon swapoff +umount uname tar cpio gzip gunzip zcat netstat ping +shutdown fdisk getty halt ifconfig init mkswap reboot route swapon swapoff

buildroot:

-

The mandatory packages +

If a toybox-based development environment is to support running +buildroot under it, the mandatory packages section of the buildroot manual lists:

-

+

which sed make bash patch gzip bzip2 tar cpio unzip rsync file bc wget -

+

(It also lists binutils gcc g++ perl python, and for debian it wants build-essential. And it wants file to be in /usr/bin because libtool breaks otherwise.)

-

Buildroot does not support a cross toolchain that lives in "/usr/bin" +

Oddly, buildroot can't NOT cross compile. Buildroot does not support a cross toolchain that lives in "/usr/bin" with a prefix of "" (if you try, and chop out the test for a blank prefix, -it dies trying to run "/usr/bin/-gcc"). But you can patch your way to -making it work if you try.

+it dies trying to run "/usr/bin/-gcc"). You can patch your way to +making it work if you try, but buildroot's developers explicitly do not +support this.

klibc:

@@ -660,7 +668,7 @@ and nobody's quite sure if the license is BSD or GPL. It inexplicably requires perl to build, and seems like an ideal candidate for replacement.

-

In addition to a C library even less capable than bionic (obsoleted by +

In addition to a C library less general-purpose than bionic (let alone musl), klibc builds a random assortment of executables to run init scripts with. There's no multiplexer command, these are individual executables:

@@ -683,26 +691,20 @@ which removes mknodes, mksyntax, sha1hash, and fixdep from the list. (And sha1hash is just an unpolished sha1sum anyway.)

The run-init command is more commonly called switch_root, nuke is just -"rm -rf -- $@", and minips is more commonly called "ps". I'm not doing aliases -for the oddball names.

- -

Yet more stale forks of dash and gzip sucked in here (see "dubious -license terms" above), adding nothing to the other projects we've looked at. -But we still need sh, gunzip, gzip, and zcat to replace this package.

+"rm -rf -- $@", and minips is more commonly called "ps": I'm not doing aliases +for these oddball names. +The "kinit" command is another gratuitous rename, it's init running as PID 1. +The halt, poweroff, and reboot commands work with it.

-

At the time I did the initial analysis toybox already had cat, chroot, dmesg, false, -kill, ln, losetup, ls, mkdir, mkfifo, readlink, rm, switch_root, sleep, sync, -true, and uname.

+

Yet more stale forks of dash and gzip got sucked in here (see "dubious +license terms" above). -

The low hanging fruit is cpio, dd, ps, mv, and pivot_root.

+

In theory "blkid" or "file" handle fstype (and df for mounted filesystems), +but we could do fstype.

-

The "kinit" command is another gratuitous rename, it's init running as PID 1. -The halt, poweroff, and reboot commands work with it.

- -

I've got mount and umount queued up already, fstype and nfsmount go with -those. (And probably smbmount and p9mount, but this hasn't got one. Those -are all about querying for login credentials, probably workable into the -base mount command.)

+

We should implement nfsmount, and probably smbmount +and p9mount even though this hasn't got one. The reason these aren't +in the base "mount" command is they interactively query login credentials.

The ipconfig command here has a built in dhcp client, so it's ifconfig and dhcpcd and maybe some other stuff.

@@ -716,12 +718,12 @@ still parses "resume=" on the command line). And yet various distros seem to make use of klibc for this. Given the history of swsusp/hibernate (and TuxOnIce -and kexec jump) I've lost track +and kexec jump...) I've lost track of the current state of the art here. Ah, Documentation/power/userland-swsusp.txt has the API docs, and here's a better tool...

-

So the list of things actually in klibc are:

+

This gives us a klibc command list:

@@ -748,29 +750,26 @@ catchsegv getconf getent iconv iconvconfig ldconfig ldd locale localedef mtrace nscd rpcent rpcinfo tzselect zdump zic
-

Of those, musl libc only implements ldd.

- -

catchsegv is a rudimentary debugger, probably out of scope for toybox.

- -

iconv has been previously discussed.

- -

iconvconfig is only relevant if iconv is user-configurable; musl uses a -non-configurable iconv.

- -

getconf is a posix utility which displays several variables from -unistd.h; it probably belongs in the development toolchain.

- -

getent handles retrieving entries from passwd-style databases -(in a rather lame way) and is trivially replacable by grep.

+

Of those, musl libc only implements ldd. Of the rest:

-

locale was discussed under posix. -localedef compiles locale definitions, which musl currently does not use.

- -

mtrace is a perl script to use the malloc debugging that glibc has built-in; -this is not relevant for musl, and would necessarily vary with libc.

- -

nscd is a name service caching daemon, which is not yet relevant for musl. -rpcinfo and rpcent are related to rpc, which musl does not include.

+
    +
  • catchsegv is a rudimentary debugger, probably out of scope for toybox.
    • +
  • iconv has been previously discussed.
    • +
  • iconvconfig is only relevant if iconv is user-configurable; musl uses a +non-configurable iconv.
    • +
  • getconf is a posix utility which displays several variables from +unistd.h; it probably belongs in the development toolchain.
    • +
  • getent handles retrieving entries from passwd-style databases +(in a rather lame way) and is trivially replacable by grep.
    • +
  • locale was discussed under posix. +localedef compiles locale definitions, which musl currently does not use.
    • +
  • mtrace is a perl script to use the malloc debugging that glibc has built-in; +this is not relevant for musl, and would necessarily vary with libc.
    • +
  • nscd is a name service caching daemon, which is not yet relevant for musl.
    • +
  • rpcinfo and rpcent are related to the Remote Procedure Calls +layer (an old sun technology used by some userspace NFS implementations), +which musl does not include and debian does not install by default.
    • +

The remaining commands involve glibc's bundled timezone database, which seems to be derived from the IANA @@ -778,14 +777,17 @@ timezone database. Unless we want to maintain our own fork of the standards body's database like glibc does, these are of no interest, but for completeness:

-

tzselect outputs a TZ variable correponding to user input. +

  • +
  • tzselect outputs a TZ variable correponding to user input. The documentation does not indicate how to use it in a script, but it seems -that Debian may have done so. -zdump prints current time in each of several timezones, optionally -outputting a great deal of extra information about each timezone. -zic converts a description of a timezone to a file in tz format.

    +that Debian may have done so.
    • +
  • zdump prints current time in each of several timezones, optionally +outputting a great deal of extra information about each timezone.
    • +
  • zic converts a description of a timezone to a file in tz format.
    • + -

    None of glibc's bundled commands are currently of interest to toybox.

    +

    We implemented getconf, and I could see maybe arguing for ncsd. +The rest are not relevant to toybox.

    @@ -1238,6 +1240,7 @@ ascii crc32 devmem fmt i2cdetect i2cdump i2cget i2cset mcookie prlimit sntp ulim blkdiscard rtcwake watchdog pwgen readelf unicode +rsync
    -- 2.39.2