From 3b41ba187f164907640af2f6e76bd51724848e3e Mon Sep 17 00:00:00 2001 From: Rob Landley Date: Wed, 27 Oct 2021 03:56:19 -0500 Subject: [PATCH] Update roadmap and mkstatus. --- scripts/mkstatus.py | 18 +-- www/roadmap.html | 302 +++++++++++++++++++++++++++++++------------- 2 files changed, 226 insertions(+), 94 deletions(-) diff --git a/scripts/mkstatus.py b/scripts/mkstatus.py index c1b325eb..e49ab2ce 100755 --- a/scripts/mkstatus.py +++ b/scripts/mkstatus.py @@ -50,12 +50,12 @@ if unknowns: print "uncategorized: %s" % " ".join(unknowns) conv = [("posix", '%%s', "[%s]"), ("lsb", '%%s', '<%s>'), - ("development", '%%s', '(%s)'), + ("development", '%%s', '(%s)'), ("toolbox", "", '{%s}'), ("klibc_cmd", "", '=%s='), ("sash_cmd", "", '#%s#'), ("sbase_cmd", "", '@%s@'), ("beastiebox_cmd", "", '*%s*'), ("tizen", "", '$%s$'), - ("shell", "", "%%%s%%"), - ("request", '%%s', '+%s+')] + ("shell", "", "%%%s%%"), ("fhs_cmd", "", '-%s-'), ("yocto", "", ".%s."), + ("request", '%%s', '+%s+')] def categorize(reverse, i, skippy=""): @@ -98,21 +98,21 @@ print "implemented=%s" % len(toystuff) outfile=open("www/status.gen", "w") outfile.write("

Status of toybox %s

\n" % version[0]); -outfile.write("

Legend: [posix] <lsb> (development) {android}\n") -outfile.write("=klibc= #sash# @sbase@ *beastiebox* $tizen$ %shell% +request+ other\n") -outfile.write("pending

\n"); +outfile.write("

Legend: %s pending

\n"%" ".join(map(lambda i: i[2]%(i[0].split("_")[0]), conv[:-1]))) outfile.write("

Completed

%s

\n" % "\n".join(done)) -outfile.write("

Partially implemented

%s

\n" % "\n".join(pend)) +outfile.write("

Partially implemented (in toys/pending)

%s

\n" % "\n".join(pend)) outfile.write("

Not started yet

%s

\n" % "\n".join(todo)) # Output unfinished commands by category outfile.write("

Categories of remaining todo items

") -for i in stuff: +for i in conv: todo = [] + i=i[0] + if i=="shell": continue for j in stuff[i]: if j in toystuff: continue if j in pending: todo.append('%s' % j) @@ -122,7 +122,7 @@ for i in stuff: k = i for j in conv: if j[0] == i: - k = j[2] % i + k = j[2] % i.split("_")[0] outfile.write("

%s

" % (i,i,k)) outfile.write(" ".join(todo)) diff --git a/www/roadmap.html b/www/roadmap.html index 41b3866a..400fa329 100644 --- a/www/roadmap.html +++ b/www/roadmap.html @@ -2,42 +2,19 @@ Toybox Roadmap -

Goals and use cases

- -

We have several potential use cases for a new set of command line -utilities, and are using those to determine which commands to implement -for Toybox's 1.0 release. (Most of these have their own section in the -status page.)

- -

The most interesting publicly available standards are POSIX-2008 -(also known as the Single -Unix Specification version 4) and the Linux Standard Base (version 4.1). -The main test harness is including toybox in Aboriginal Linux and if that can -build itself using the result to build Linux From Scratch (version 6.8). -We also aim to replace Android's Toolbox.

- -

At a secondary level we'd like to meet other use cases. We've analyzed -the commands provided by similar projects (klibc, sash, sbase, embutils, -nash, and beastiebox), along with various vendor configurations of busybox, -and some end user requests.

- -

Finally, we'd like to provide a good replacement for the Bash shell, -which was the first program Linux ever ran and remains the standard shell -of Linux no matter what Ubuntu says. This doesn't mean including the full -set of Bash 4.x functionality, but does involve {various,features} <(beyond) -posix.

- -

See the status page for the categorized command list -and progress towards implementing it. There's also a -historical todo list from the project's 2011 relaunch.

+

Roadmap sections

+ +

Introduction

+ +

We have several potential use cases for a new set of command line +utilities, and are using those to determine which commands to implement +for Toybox's 1.0 release. Most of these have their own section in the +status page, showing current progress towards +commplation.

+ +

The most interesting publicly available standards are POSIX-2008 (also +known as SUSv4), the Linux Standard Base (version 4.1), and the official +Linux man pages, all +of which include commands we've decided not implement and exclude +commands or features we have.

+ +

The most thorough real world test (other than a large interactive +userbase) is using toybox in a +development environment by providing the commands for a build system such as +Aboriginal +Linux, having it rebuild itself from source code, and using the result +to build Linux From Scratch. +The current goal is to use mkroot +plus musl-cross-make to hermetically build +AOSP.

+ +

We've also checked what commands were provided by similar projects +(klibc, sash, sbase, embutils, +nash, and beastiebox), looked at various vendor configurations of busybox, +and collected end user requests.

+ +

Finally, we'd like to provide a good replacement for the Bash shell, +which was the first program Linux ever ran and remains the standard shell +of Linux (no matter what Ubuntu says). This doesn't necessarily mean including +every last Bash 5.x feature, but does involve {various,features} <(beyond) +posix.

+ +

See the status page for the categorized command list +and progress towards implementing it. There's also a +historical todo list from the project's 2011 relaunch.

+

Use case: standards compliance.

@@ -53,40 +70,52 @@ and progress towards implementing it. There's also a

The best standards describe reality rather than attempting to impose a new one. A good standard should document, not legislate. Standards which document existing reality tend to be approved by -more than one standards body, such ANSI and ISO both approving C99. That's why -the IEEE POSIX committee's 2008 standard, the Single Unix Specification version -4, and the Open Group Base Specification edition 7 are all the same standard -from three sources, but most people just call it "posix" (portable operating -system derived from unix). It's available online in full, and may be downloaded as a tarball... -with a caveat.

- -

Although previous versions of Posix have their own -stable URLs (where you can still find -SUSv3 and -SUSv2), -the 2008 release of SUSv4 was replaced by a 2013 release also claiming -to be SUSv4, then again by a 2018 release still at the same URL. Similarly, -the other version numbers claim not to have changed, but instead adopted some -sort of "Windows 95" naming scheme ("The Open Group Base Specifications Issue -7, 2018 edition"). Since a moving target isn't a standard, we've stuck -with the 2008 version and ignored whatever changes they make -until they stop this forced-upgrade-behind-your back nonsense. -Luckily you can still find the original content -here. -(We haven't changed the URLs in each command to the longer version yet, -but can if conflicts arise.)

+more than one standards body, such ANSI and ISO both approving C99. That's why IEEE 1003.1-2008, +the Single Unix Specification version 4, and the Open Group Base Specification +edition 7 are all the same standard from three sources, but most people just +call it "posix" (portable operating system derived from unix). +It's available online in full, and may be downloaded as a tarball. +Previous versions (SUSv3 and +SUSv2) +are also available. +(Note: +Posix +2008 was reissued in 2013 and 2018, the first was minor wordsmithing +with no behavioral changes, the second was to renew a ten year timeout +to still be considered a "current standard" by some government regulations. +It's still posix-2008/SUSv4/issue 7.)

Why not just use posix for everything?

-

Unfortunately posix describes an incomplete subset of reality, -lacking any mention of commands such as init or mount required to -actually boot a system. It describes logname but not login. It provides ipcrm +

Unfortunately, Posix describes an incomplete subset of reality, because +it was designed to. It started with proprietary unix vendors collaborating to +describe the functionality their fragmented APIs could agree on, which was then +incorporated into US federal procurement standards +as a compliance requirement +for things like navy contracts, giving large corporations +like IBM and Microsoft millions of dollars of incentive +to punch holes in the standard big enough to drive +Windows NT and +OS/360 through. +When open source projects like Linux started developing on the internet +(accelerated by the 1993 relaxation of the National Science Foundation's +"Acceptable Use Policy" allowing everyone to connect to the internet, +previously restricted to approved government/military/university organizations), +Posix ignored +the upstarts and Linux eventually +returned the favor, +leaving Posix behind.

+ +

The result is a "standard" that lacks any mention of commands like +"init" or "mount" required to actually boot a system. +It describes logname but not login. It provides ipcrm and ipcs, but not ipcmk, so you can use System V IPC resources but not create them. And widely used real-world commands such as tar and cpio (the basis of initramfs and RPM) which were present in earlier versions of the standard have been removed, while obsolete commands like cksum, compress, sccs and uucp remain with no mention of modern counterparts -like crc32/sha1sum, gzip/xz, svn/git or scp/rsync. Meanwhile the commands +like crc32/sha1sum, gzip/xz, svn/git or scp/rsync. Meanwhile posix' description +of the commands themselves are missing dozens of features and specify silly things like ebcdic support in dd or that wc should use %d (not %lld) for byte counts. So we have to extensively filter posix to get a useful set of recommendations.

@@ -100,18 +129,18 @@ val what), fortran support (asa fort77), and batch processing support (batch qalter qdel qhold qmove qmsg qrerun qrls qselect qsig qstat qsub).

Some commands are for a compiler toolchain (ar c99 cflow ctags cxref gencat -iconv lex m4 make nm strings strip tsort yacc), which is outside of toybox's +iconv lex m4 make nm strings strip tsort yacc) which is outside of toybox's mandate and should be supplied externally. (Again, some of these may be revisited later, but not for toybox 1.0.)

Some commands are part of a command shell, and can't be implemented as separate executables (alias bg cd command fc fg getopts hash jobs kill read -type ulimit umask unalias wait). These may be revisited as part of a built-in +type ulimit umask unalias wait). These may be revisited as part of the built-in toybox shell, but are not exported into $PATH via symlinks. (If you fork a child process and have it "cd" then exit, you've accomplished nothing.) Again, what posix provides is incomplete: a shell also needs exit, if, while, for, case, export, set, unset, trap, exec... (And for bash compatibility -function, source...)

+function, source, declare...)

A few other commands are judgement calls, providing command-line internationalization support (iconv locale localedef), System V inter-process @@ -122,7 +151,8 @@ a command line email client, and "lp" submits files for printing to... what exactly? (cups?) The standard defines crontab but not crond. What is pathchk supposed to be portable _to_? (Linux accepts 255 byte path components with any char except NUL or / and no max length on the total path, and -EXPLICITLY doesn't care if it's an invalid utf8 sequence.)

+EXPLICITLY +doesn't care if it's an invalid utf8 sequence.)

Removing all of that leaves the following commands, which toybox should implement:

@@ -143,32 +173,33 @@ who xargs zcat

One attempt to supplement POSIX towards an actual usable system was the Linux Standard Base. Unfortunately, the quality of this "standard" is -fairly low.

+fairly low, largely due to the Free Standards Group that maintained it +being consumed by the Linux Foundation in 2007.

-

POSIX allowed its standards process to be compromised -by leaving things out, thus allowing IBM mainframes and Windows NT to drive -a truck through the holes and declare themselves compilant. But it means what -they DID standardize tends to be respected (if sometimes obsolete).

- -

The Linux Standard Base's failure mode is different, they respond to +

Where POSIX allowed its standards process to be compromised +by leaving things out (but what +they DID standardize tends to be respected, if sometimes obsolete), +the Linux Standard Base's failure mode is different. They respond to pressure by including anything their members pay them enough to promote, such as allowing Red Hat to push RPM into the standard even though all sorts of distros (Debian, Slackware, Arch, -Gentoo) don't use it and never will. This means anything in the LSB is +Gentoo, Android) don't use it and never will. This means anything in the LSB is at best a suggestion: arbitrary portions of this standard are widely ignored.

-

The community perception seems to be that the Linux Standard Base is +

The community perception +seems to be that the Linux Standard Base is the best standard money can buy, I.E. the Linux Foundation is supported by financial donations from large companies and the LSB represents the interests -of those donors more than technical merit. (The Linux Foundation, which +of those donors regardless of technical merit. (The Linux Foundation, which maintains the LSB, isn't a 501c3. It's a 501c6, the -same kind of legal entity as the Tobacco Institute and Microsoft's old -"Don't Copy That Floppy" program.) Debian officially +same kind of legal entity as the Tobacco Institute and +Microsoft's +old "Don't Copy That Floppy" program.) Debian officially washed its hands of LSB when 5.0 came out in 2015, and no longer even pretends to support it (which may affect -Debian derivatives like Ubuntu and Knoppix). Toybox hasn't moved to 5.0 for +Debian derivatives like Ubuntu and Knoppix). Toybox has stayed on 4.1 for similar reasons.

That said, Posix by itself isn't enough, and this is the next most @@ -213,12 +244,20 @@ su sync tar umount useradd userdel usermod zcat

-

IETF RFCs

+

IETF RFCs and Man Pages

+ +

They're very nice. There's thousands of them.

Discussion of standards wouldn't be complete without the Internet -Engineering Task Force's "Request For Comments" collection.

+Engineering Task Force's "Request For Comments" collection and Michael Kerrisk's +Linux man-pages project, +but neither helps us select which commands to include.

+ +

The man pages website includes the commands in git, yum, perf, postgres, +flatpack... Great for examining the features of a command you've +already decided to include, useless for deciding _what_ to include.

-

These are more about protocols than commands. The noise level is +

The RFCs are more about protocols than commands. The noise level is extremely high: there's thousands of RFCs, many describing a proposed idea that never took off, and less than 1% of the resulting documents are currently relevant to toybox. And the documents are numbered based on the @@ -229,9 +268,9 @@ have developed a certain amount of they document is to find an earlier version to read first.

That said, RFC documents can be useful (especially for networking protocols) -and the three URL templates the recommended starting files +and the four URL templates the recommended starting files for new commands (toys/example/skeleton.c or toys/example/hello.c depending on how much -plumbing you want to start with) provide point to are posix, lsb, and +plumbing you want to start with) provide point to are posix, lsb, man, and rfc pages.

@@ -290,7 +329,7 @@ still needs the following busybox commands for which toybox does not yet supply adequate replacements:

-awk dd diff expr fdisk ftpd gzip less route sh sha512sum tr unxz vi wget xzcat +awk dd diff expr fdisk ftpd gzip less route sh tr unxz vi wget xzcat

All of those except awk, ftpd, and less have partial implementations @@ -473,21 +512,32 @@ and hexdump that are actually used to build.)

Use case: Tizen Core

-

The Tizen project has expressed a desire to eliminate GPLv3 software -from its core system, and is installing toybox as +

A side effect of the Linux Foundation following the money to the +exclusion of all else is they "support" their donors' myriad often +contradictory pet projects with elaborate announcements and press releases. +Long ago when Nokia's Maemo merged +with Intel's Moblin to form MeeGo, there were believable statements +about unifying fragmented vendor efforts. Then MeeGo merged with +LiMo to +form Tizen, +which became a Samsung-only project (that still ships inside televisions, +but was otherwise subsumed into Android GO).

+ +

Along the way, the Tizen project expressed a desire to eliminate GPLv3 software +from its core system, and in installing toybox as part of this process.

-

They have a fairly long list of new commands they'd like to see in toybox:

+

They had a fairly long list of new commands they wanted to see in toybox:

arch base64 users unexpand shred join csplit hostid nproc runcon sha224sum sha256sum sha384sum sha512sum sha3sum mkfs.vfat fsck.vfat -dosfslabel uname stdbuf pinky diff3 sdiff zcmp zdiff zegrep zfgrep zless zmore +dosfslabel uname pinky diff3 sdiff zcmp zdiff zegrep zfgrep zless zmore
-

In addition, they'd like to use several commands currently in pending:

+

In addition, they wanted to use several commands then in pending:

@@ -496,8 +546,89 @@ tar diff printf wget rsync fdisk vi less tr test stty fold expr dd

Also, tizen uses a different Linux Security Module called SMACK, so -many of the SELinux options ala ls -Z need smack alternatives in an -if/else setup.

+many of the SELinux options ala ls -Z needed smack alternatives in an +if/else setup. (We added lib/lsm.h to abstract this.)

+ +
+

Use case: Yocto

+ +

Another project the Linux Foundation got paid to appreciate is Yocto, +which was designed to fix the ongoing proprietary fragmentation problem +(now in Linux build systems instead of propreitary unixes) by being the +build system equivalent of a glue trap. While proclaiming that having the +"minimum level of standardization" contributes to a "strong ecosystem", +Yocto uses a "layered" +design where everybody who touches it is encouraged to add more and more layers +of metadata on top of what came before, until they wind up using repo just to manage +the layers of metadata (let alone their contents). But (and this is the +important bit), all these dispirate forks are called yocto and built on +top of giant piles of code the Linux Foundation can take credit for. +(Although really it's a reskin of OpenEmbedded.)

+ +

Yocto's "core-image-minimal" target (only 3,106 build stages in the 3.3 +release, which believe it or not is +an improvement) builds a busybox-based system with the following commands:

+ +
+ +addgroup adduser ascii ash awk base32 basename blkid bunzip2 bzcat bzip2 cat +chattr chgrp chmod chown chroot chvt clear cmp cp cpio crc32 cut date dc dd +deallocvt delgroup deluser depmod df diff dirname dmesg dnsdomainname du +dumpkmap dumpleases echo egrep env expr false fbset fdisk fgrep find flock +free fsck fstrim fuser getopt getty grep groups gunzip gzip head hexdump +hostname hwclock id ifconfig ifdown ifup insmod ip kill killall klogd less +ln loadfont loadkmap logger logname logread losetup ls lsmod lzcat md5sum +mesg microcom mkdir mkfifo mknod mkswap mktemp modprobe more mount mountpoint +mv nc netstat nohup nproc nslookup od openvt patch pgrep pidof pivot_root +printf ps pwd rdate readlink realpath reboot renice reset resize rev rfkill +rm rmdir rmmod route run-parts sed seq setconsole setsid sh sha1sum sha256sum +shuf sleep sort start-stop-daemon stat strings stty sulogin swapoff swapon +switch_root sync sysctl syslogd tail tar tee telnet test tftp time top touch +tr true ts tty udhcpc udhcpd umount uname uniq unlink unzip uptime users +usleep vi watch wc wget which who whoami xargs xzcat yes zcat + +
+ + + +
Filesystem Hierachy Standard +

Filesystem Hierarchy Standard:

+ +

Another standard captured by the Linux Foundation. (At least the +links to this one didn't go 404 the +instant they took it over). Of historical interest due to what it +managed to achieve before they chased away the hobbyists maintaining it. +Only one version (3.0 in 2015) has been released since the Linux Foundation +consumed the FHS. The previous release, Version 2.3, was released in 2004. +The Linux Foundation did not retain earlier versions. The contents of +the relevant sections appear identical between the two versions (the +Linux Foundation added section numbers, that appears to be it).

+ +

FHS 3.0 +section 3.4.2 requires commands to be in the /bin directory, and then 3.4.3 +has an optional list, +and then 3.16.2 and 3.16.3 similarly cover /sbin. (There are linux +specific sections in 6.1.2 and 6.1.6 but everything in them is obsolete.) +The contents appear to be identical between 2.3 and 3.0: the only change +the Linux Foundation made here was to add section numbers.

+ +

The /bin options include csh but not bash, and ed but not vi, which +is crazy. The /sbin options has update which seems obsolete (filesystem +buffers haven't needed a userspace process to flush them for DECADES), +fastboot and fasthalt (reboot and halt have -nf), and +fsck.* and mkfs.* that don't actually specify specific filesystems. +Removing that gives us:

+ +
+ +cat chgrp chmod chown cp date dd df dmesg echo false hostname kill ln +login ls mkdir mknod more mount mv ps pwd rm rmdir sed sh stty su sync true +umount uname +tar cpio gzip gunzip zcat netstat ping +shutdown +fdisk fsck getty halt ifconfig init mkfs mkswap reboot route swapon swapoff + +

buildroot:

@@ -1094,7 +1225,7 @@ arp makedevs sysctl killall5 crond crontab deluser last mkpasswd watch blockdev rpm2cpio arping brctl dumpleases fsck tcpsvd tftpd factor fallocate fsfreeze inotifyd lspci nbd-client partprobe strings -base64 mix +base32 base64 mix reset hexedit nsenter shred fsync insmod ionice lsmod lsusb rmmod vmstat xxd top iotop lsof ionice compress dhcp dhcpd addgroup delgroup host iconv ip @@ -1106,6 +1237,7 @@ kexec ascii crc32 devmem fmt i2cdetect i2cdump i2cget i2cset mcookie prlimit sntp ulimit uuidgen dhcp6 ipaddr iplink iproute iprule iptunnel cd exit toysh bash traceroute6 blkdiscard rtcwake watchdog +pwgen readelf unicode -- 2.39.2