libk  Check-in [b590aba21d]

#!/usr/bin/env bash
export out=${out:-out}
. global/common.sh
noimpl() {
	say "$1 is not currently implemented on your system. an exciting chance for you to contribute, no?"
	exit 1
}
timestamp() { date "+%H:%M:%S %A"; }

if test "$os$arch$bits" = ""; then
	say "set the following environment variables to the appropriate atoms describing your system. for help determining the appropriate atoms, see libk.md"
	say ' - $os={lin|fbsd|hai|osx…}'
	say ' - $arch={x86|arm|ia64|mips…}'
	say ' - $bits={|32|64…}'
	exit 1
................................................................................
has clang && _cc=clang 
has gcc && _cc=gcc # prefer gcc
cc=${cc:-$_cc}
has m4 && _m4=m4
m4=${m4:-$_m4}
has nasm && asm=nasm
has yasm && asm=yasm # prefer yasm


export gen=${gen:-gen}
library=${library:-static} # {static|shared|both}
export verbose=${verbose:-quiet} # {no|quiet|loud}

doc=${doc:-yes}
export doc_html=${doc_html:-yes}
export doc_pdf=${doc_pdf:-yes}
................................................................................

macro_compile_env="-Datom_target_arch=$arch -Datom_target_os=$os -Dtarget_posix=$posix -Dtarget_unix=$unix"

if test "$bits" != ""; then
	macro_compile_env="$macro_compile_env -Datom_target_bits=$bits"
fi

if test "$COLORTERM" != ""; then
	announce() {
		test "$verbose" = "silent" && return
		color="$1" cmd="$2"; shift 2;
		printf " → [38;5;$color;1m$cmd"
		for a in "$@"; do
			if test ${a:0:1} = "-"; then
				printf " $a";
			else
				printf " $a";
			fi
		done
		echo
	}
else
	announce() { shift; echo " --> " $@; }
fi

# the following function is called to report a command invocation
# the person compiling the library. the first argument should be
# an ansi format string; this is used to color-code the tool being
# launched and thus should be different for each one.
report() { announce $@; shift; $@; }
comp_mac() {
	local src=$1
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi 
	fi
	             $m4 $macro_compile_env -I "$gen" $flags "$src"  > "$output"
	announce 207 $m4 $macro_compile_env -I "$gen" $flags "$src" \> "$output"
	# yes, this is incredibly stupid. if you know a better way, feel
	# free to submit a fix. the problem is there's no way to pass >
	# to report in such a way that it'll do the right thing, and if
	# you just write > it redirects *report's* output, instead of
	# m4's. piece of shit that it is, m4 doesn't have any way to emit
	# output into a fille - stdout only apparently. tl;dr i hate bash.
}
................................................................................
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi
	fi
	report 198 $asm $flags "-f$bin_fmt" -i "$gen" "$src" -o "$output";
}
comp_co()  { comp_c $1 $2 "-c -fPIC"; }
comp_c(){
	local src=$1
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi
	fi
	# only rebuild the file if the source file is newer
	report 213 $cc $src $3 -std=c11 -isystem "$out" -isystem "$gen" -isystem "arch/" -nostdlib "-L$out" "-o$output"
}

scan() { find "$1" -name "$2"; }

say "commencing libk build $build at $(timestamp)"
# set -x

# get type data
mkdir -p $gen
$cc -D_emit_m4_include arch/typesize.c -o $gen/typesize 
$gen/typesize > gen/typesize.m
................................................................................
grep -h "#[ 	]*define[ 	]\+E" $p_headers_errno |
	sed 's;^#[\t ]*define[\t ]\+\(E[A-Z0-9]\+\).*$;k_platform_error_\1 \1;' >\
	$gen/error_names.tbl
cat $p_headers_errno $gen/error_names.tbl | cpp -P >$gen/error_numbers.tbl
awk -f arch/errtbl.awk <$gen/error_numbers.tbl >$gen/error_table.h

# generate symbol tables for error handling functions
mkdir -p "$out/k"
awk -f global/gen-conds.awk <global/modules >$out/k/internal.egroup.h
awk -f global/gen-ident.awk <global/modules >$gen/internal.ident.c
comp_co $gen/internal.ident.c $out/internal.ident.o

# first pass: copy all headers into place,
# including ones we need to generate

for mod in ${modules[@]}; do
	for h in $(scan $mod '*.h'); do
		base=$(basename $h)
		cp "$h" "$out/k/$base"
	done

	for h in $(scan $mod '*.h.m'); do
		base=$(basename $h)
		dest=${base%%.m}
		comp_mac "$h" "$out/k/$dest"
	done
done

# second pass: generate manpage, html, and pdf
# versions of the documentation from the md src

if test "$doc" = "yes"; then
	global/build-manpage.sh libk.md
	for mod in ${modules[@]}; do
		for doc in $(scan $mod '*.md'); do
			base="$(basename $doc)"
			stem="${base%%.md}"
			report 177 global/build-manpage.sh "$doc"
		done
	done
fi

# third pass: compile sources and save the
# resulting object files to $out, tracking
# which is a runtime or function unit. exe's
# will not be compiled until a later pass

fn_objects=()
rt_objects=()
data_objects=( $out/internal.ident.o )
for mod in ${modules[@]}; do
	for fn in $(scan $mod '*.fn.c'); do
		base="$(basename $fn)"
		dest="$out/$mod.${base%%.c}.o"
		comp_co "$fn" "$dest"
		fn_objects+=("$dest")
	done

	for rt in $(scan $mod '*.rt.c'); do
		base="$(basename $rt)"
		dest="$out/$mod.${base%%.c}.o"
		comp_co "$rt" "$dest"
		rt_objects+=("$dest")
	done

	for fn in $(scan $mod "*.fn.$target.s"); do
		base="$(basename $fn)"
		dest="$out/$mod.${base%%.s}.o"
		comp_asm "$fn" "$dest"
		fn_objects+=("$dest")
	done

	for rt in $(scan $mod "*.rt.$target.s"); do
		base="$(basename $rt)"
		dest="$out/$mod.${base%%.s}.o"
		comp_asm "$rt" "$dest"
		rt_objects+=("$dest")
	done
done

# fourth pass: link the libraries that are
# configured to be built

if test $build_static_library == yes; then
	for obj in ${fn_objects[@]} ${rt_objects[@]} ${data_objects[@]}; do
		report 120 ar rc $out/libk.a $obj
	done
	report 105 ranlib $out/libk.a
fi

if test $build_shared_library == yes; then
	report 216 ld -r ${rt_objects[@]} -o $out/boot.o
	report 216 ld -shared ${fn_objects[@]} -o $out/libk.so
fi

# fifth pass: compile the executable tools
# against the libraries created in pass 5

for mod in ${modules[@]}; do
	for exe in $(scan $mod '*.exe.c'); do
		base="$(basename $exe)"
		dest="$out/$mod.${base%%.exe.c}"
		if test $build_shared_library == yes; then
			comp_c "$out/boot.o $exe" "$dest" -lk
		else
			comp_c "$exe" "$dest" -lk
		fi
	done
done

set +x
say "all passes finished; build $build complete at $(timestamp)"

#!/usr/bin/env bash
export to=${to:-out}
. global/common.sh






if test "$os$arch$bits" = ""; then
	say "set the following environment variables to the appropriate atoms describing your system. for help determining the appropriate atoms, see libk.md"
	say ' - $os={lin|fbsd|hai|osx…}'
	say ' - $arch={x86|arm|ia64|mips…}'
	say ' - $bits={|32|64…}'
	exit 1
................................................................................
has clang && _cc=clang 
has gcc && _cc=gcc # prefer gcc
cc=${cc:-$_cc}
has m4 && _m4=m4
m4=${m4:-$_m4}
has nasm && asm=nasm
has yasm && asm=yasm # prefer yasm
export cc
export m4
export gen=${gen:-gen}
library=${library:-static} # {static|shared|both}
export verbose=${verbose:-quiet} # {no|quiet|loud}

doc=${doc:-yes}
export doc_html=${doc_html:-yes}
export doc_pdf=${doc_pdf:-yes}
................................................................................

macro_compile_env="-Datom_target_arch=$arch -Datom_target_os=$os -Dtarget_posix=$posix -Dtarget_unix=$unix"

if test "$bits" != ""; then
	macro_compile_env="$macro_compile_env -Datom_target_bits=$bits"
fi
























comp_mac() {
	local src=$1
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi 
	fi
	          $m4 $macro_compile_env -I "$gen" $flags "$src"  > "$output"
	announce  $m4 $macro_compile_env -I "$gen" $flags "$src" \> "$output"
	# yes, this is incredibly stupid. if you know a better way, feel
	# free to submit a fix. the problem is there's no way to pass >
	# to report in such a way that it'll do the right thing, and if
	# you just write > it redirects *report's* output, instead of
	# m4's. piece of shit that it is, m4 doesn't have any way to emit
	# output into a fille - stdout only apparently. tl;dr i hate bash.
}
................................................................................
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi
	fi
	report $asm $flags "-f$bin_fmt" -i "$gen" "$src" -o "$output";
}
comp_co()  { comp_c $1 $2 "-c -fPIC"; }
comp_c(){
	local src=$1
	local output=$2
	local flags=$3
	if test -e "$output"; then
		if test ! "$output" -ot "$src"; then
			return
		fi
	fi
	# only rebuild the file if the source file is newer
	report $cc $src $3 -std=c11 -isystem "$to" -isystem "$gen" -isystem "arch/" -nostdlib "-L$to" "-o$output"
}



say "commencing libk build $build at $(timestamp)"
# set -x

# get type data
mkdir -p $gen
$cc -D_emit_m4_include arch/typesize.c -o $gen/typesize 
$gen/typesize > gen/typesize.m
................................................................................
grep -h "#[ 	]*define[ 	]\+E" $p_headers_errno |
	sed 's;^#[\t ]*define[\t ]\+\(E[A-Z0-9]\+\).*$;k_platform_error_\1 \1;' >\
	$gen/error_names.tbl
cat $p_headers_errno $gen/error_names.tbl | cpp -P >$gen/error_numbers.tbl
awk -f arch/errtbl.awk <$gen/error_numbers.tbl >$gen/error_table.h

# generate symbol tables for error handling functions
mkdir -p "$to/k"
awk -f global/gen-conds.awk <global/modules >$to/k/internal.egroup.h
awk -f global/gen-ident.awk <global/modules >$gen/internal.ident.c
comp_co $gen/internal.ident.c $to/internal.ident.o

# first pass: copy all headers into place,
# including ones we need to generate

for mod in ${modules[@]}; do
	for h in $(scan $mod '*.h'); do
		base=$(basename $h)
		cp "$h" "$to/k/$base"
	done

	for h in $(scan $mod '*.h.m'); do
		base=$(basename $h)
		dest=${base%%.m}
		comp_mac "$h" "$to/k/$dest"
	done
done

# second pass: generate manpage, html, and pdf
# versions of the documentation from the md src

if test "$doc" = "yes"; then
	global/build-manpage.sh libk.md
	for mod in ${modules[@]}; do
		for doc in $(scan $mod '*.md'); do
			base="$(basename $doc)"
			stem="${base%%.md}"
			report global/build-manpage.sh "$doc"
		done
	done
fi

# third pass: compile sources and save the
# resulting object files to $to, tracking
# which is a runtime or function unit. exe's
# will not be compiled until a later pass

fn_objects=()
rt_objects=()
data_objects=( $to/internal.ident.o )
for mod in ${modules[@]}; do
	for fn in $(scan $mod '*.fn.c'); do
		base="$(basename $fn)"
		dest="$to/$mod.${base%%.c}.o"
		comp_co "$fn" "$dest"
		fn_objects+=("$dest")
	done

	for rt in $(scan $mod '*.rt.c'); do
		base="$(basename $rt)"
		dest="$to/$mod.${base%%.c}.o"
		comp_co "$rt" "$dest"
		rt_objects+=("$dest")
	done

	for fn in $(scan $mod "*.fn.$target.s"); do
		base="$(basename $fn)"
		dest="$to/$mod.${base%%.s}.o"
		comp_asm "$fn" "$dest"
		fn_objects+=("$dest")
	done

	for rt in $(scan $mod "*.rt.$target.s"); do
		base="$(basename $rt)"
		dest="$to/$mod.${base%%.s}.o"
		comp_asm "$rt" "$dest"
		rt_objects+=("$dest")
	done
done

# fourth pass: link the libraries that are
# configured to be built

if test $build_static_library == yes; then
	for obj in ${fn_objects[@]} ${rt_objects[@]} ${data_objects[@]}; do
		report ar rc $to/libk.a $obj
	done
	report ranlib $to/libk.a
fi

if test $build_shared_library == yes; then
	report ld -r ${rt_objects[@]} -o $to/boot.o
	report ld -shared ${fn_objects[@]} -o $to/libk.so
fi

# fifth pass: compile the executable tools
# against the libraries created in pass 5

for mod in ${modules[@]}; do
	for exe in $(scan $mod '*.exe.c'); do
		base="$(basename $exe)"
		dest="$to/$mod.${base%%.exe.c}"
		if test $build_shared_library == yes; then
			comp_c "$to/boot.o $exe" "$dest" -lk
		else
			comp_c "$exe" "$dest" -lk
		fi
	done
done

set +x
say "all passes finished; build $build complete at $(timestamp)"

#!/usr/bin/env bash
out=${out:-out}
gen=${gen:-gen}
. global/common.sh

say "cleaning libk build artifacts"

set -x
rm -r $out
rm -r $gen

#!/usr/bin/env bash
to=${to:-out}
gen=${gen:-gen}
. global/common.sh

say "cleaning libk build artifacts"


report rm -r $to
report rm -r $gen

reqpack cmark "generate documentation"
check gen "a directory for generated build dependencies"

file="$1"
filename="$(basename "$file")"
stem=${filename%%.md}

mandest=${out_manpage:-"$out/doc/man"}
htmldest=${out_html:-"$out/doc/html"}
pdfdest=${out_pdf:-"$out/doc/pdf"}

section=${manual_section:-4}
man="$mandest/$stem.$section"
html="$htmldest/$stem.html"
pdf="$pdfdest/$stem.pdf"
fmt="$gen/$stem.tmp"

reqpack cmark "generate documentation"
check gen "a directory for generated build dependencies"

file="$1"
filename="$(basename "$file")"
stem=${filename%%.md}

mandest=${manpage_to:-"$to/doc/man"}
htmldest=${html_to:-"$to/doc/html"}
pdfdest=${pdf_to:-"$to/doc/pdf"}

section=${manual_section:-4}
man="$mandest/$stem.$section"
html="$htmldest/$stem.html"
pdf="$pdfdest/$stem.pdf"
fmt="$gen/$stem.tmp"

check() {
	var=$1
	test "${!var}" == "" || return 0
	say "we were not able to detect a default value for the configuration variable \$$var. please set this variable to $2 and try again."
	exit 1
}

test "$out" = "" && {
	say "\$out environment variable must be set to your build directory - are you running this script by hand? run ./build.sh in the root directory instead!"
	exit 2
}

reqpack() {
	if ! has "$1"; then
		say "to $2 for libk, install the $1 package and try again"
		exit 3
	fi
}

check() {
	var=$1
	test "${!var}" == "" || return 0
	say "we were not able to detect a default value for the configuration variable \$$var. please set this variable to $2 and try again."
	exit 1
}

test "$to" = "" && {
	say "\$to environment variable must be set to your build directory - are you running this script by hand? run ./build.sh in the root directory instead!"
	exit 2
}

reqpack() {
	if ! has "$1"; then
		say "to $2 for libk, install the $1 package and try again"
		exit 3
	fi
}

noimpl() {
	say "$1 is not currently implemented on your system. an exciting chance for you to contribute, no?"
	exit 1
}

timestamp() { date "+%H:%M:%S %A"; }

scan() { find "$1" -name "$2"; }

if test "$COLORTERM" != ""; then
	announce() {
		test "$verbose" = "silent" && return
		local cmd="$1"; shift;
		local color=225;
		case "$cmd" in
			rm) color=196;; $asm) color=198;;
			$cc) color=213;; ar) color=120;;
			ranlib) color=105;; ld) color=216;;
			global/build-manpage.sh) color=177;;
			$m4) color=207;;
		esac

		printf " → [38;5;$color;1m$cmd"
		for a in "$@"; do
			if test ${a:0:1} = "-"; then
				printf " $a";
			else
				printf " $a";
			fi
		done
		echo
	}
else
	announce() { shift; echo " --> " $@; }
fi

# the following function is called to report a command invocation
# the person compiling the library. the first argument should be
# an ansi format string; this is used to color-code the tool being
# launched and thus should be different for each one.

report() { announce $@; $@; }

other assemblers will probably be necessary for the more exotic targets, however.

# repository structure

libk uses a strict directory structure for code, and deviations from this structure will not be tolerated without extremely good reason.

total segregation is maintained between source code, temporary files, and output objects. source is found in module directories (`k*/`). the destination for temporary files and output objects are retargetable via the environment variables `gen= out=`, but default to `gen/` and `out/`, which are excluded from repo with fossil's `ignore-glob` settingapproval of the maintainer herself.

all libk code is dispersed into modules: `kcore` for internals, `kio` for I/O, `kgraft` for binary packing, etc. each module has a folder in the root directory. (libk does not have submodules.) inside each module's directory should be a header with the same name as the module (see **naming conventions** above).

each function should be kept in a separate file within its module's directory. the file's name should consist of the dot-separated fields [name, class, "c"] for C sources, or [name, class, arch, OS, bits, format, "s"] for assembly sources, where "name" is the name of the function without the module prefix and "class" is `rt` if the file is part of the libk runtime, or `fn` otherwise. this distinction is necessary because while the static library `libk.a` can include runtime objects, the shared library `libk.so` cannot. examples:

 * a C file in the module `kstr` named `kscomp` would be named `kstr/comp.fn.c`
 * a runtime assembly file called `boot` in the module `kcore` for x86-64 linux would be named `kcore/boot.rt.x86.lin.64.s`
 * the 32-bit x86 haiku version of a function called `kiowrite` defined in assembly would be named `kio/write.fn.x86.hai.32.s`.

each module should have a header named the same thing as the module except without the `k` prefix. (e.g. the header for `kio` is `kio/io.h`) located in its folder. this is the header that the end-user will be importing, and should handle any user-defined flags to present the API the user has selected.

each module should contain a markdown file. this file's name should be the name of the parent directory suffixed with `.md`; for instance, `kterm` should contain the file `kterm/kterm.md`. this file should document the module as thoroughly as possible 

each module may contain any number of files of the name `*.exe.c`. this files will be treated as *tools* by the build system and compiled as executables, rather than libraries. they should be compiled to `out/$module.$tool`

the repository root and each module may also contain the directory `misc`. this directory may be used to store miscellaneous data such as ABI references, developer discussions, and roadmaps. if the `misc` directory is deleted, this must not affect the library or build system's function in any way - that is, nothing outside a `misc` folder may reference a `misc` folder or anything inside it, including documentation. the `misc` directory should be removed when its contents are no longer needed. in most cases, the repository wiki and forum should be used instead of the `misc` folder.

the folder `arch` in the root of the repository contains syscall tables and ABI implementations for various architectures.

# build system

................................................................................

 * `os={atom}` - an atom representing the operating system you are building libk for
 * `arch={atom}` - an atom representing the processor architecture you are building libk for
 * `bits={atom}` - if your processor has multiple variants with different word lengths (such as x86-32 vs. x86-64), specify the word length in this variable; otherwise, leave it unset.

further optional variables may be set to control the build process and determine what targets it produces.
 * `library=static {static|shared|both}` - this variable controls whether the build process will produce `libk.a`, `libk.so`, or both.
 * `out=out {path}` - an alternate path to store build artifacts in
 * `gen=gen {path}` - an alternate path to store generated source in
 * `cc=<autodetect> {executable}` - the compiler to compile C sources with
 * `m4=<autodetect> {executable}` - the m4 binary to compile the macro sources with
 * `asm=<autodetect> {executable}` - the assembler to assemble the assembly listings with. it must take Intel-syntax input and handle nasm-style macros. only `yasm` and `nasm` are likely to be viable.
 * `doc=yes {yes|no}` - whether to typeset the documentation (very slow with all three formats set to "yes")
 * `doc_html=yes {yes|no}` - enable or disable html output of the documentation
 * `doc_pdf=yes {yes|no}` - enable or disable pdf output of the documentation

other assemblers will probably be necessary for the more exotic targets, however.

# repository structure

libk uses a strict directory structure for code, and deviations from this structure will not be tolerated without extremely good reason.

total segregation is maintained between source code, temporary files, and output objects. source is found in module directories (`k*/`). the destination for temporary files and output objects are retargetable via the environment variables `gen= to=`, but default to `gen/` and `out/`, which are excluded from repo with fossil's `ignore-glob` settingapproval of the maintainer herself.

all libk code is dispersed into modules: `kcore` for internals, `kio` for I/O, `kgraft` for binary packing, etc. each module has a folder in the root directory. (libk does not have submodules.) inside each module's directory should be a header with the same name as the module (see **naming conventions** above).

each function should be kept in a separate file within its module's directory. the file's name should consist of the dot-separated fields [name, class, "c"] for C sources, or [name, class, arch, OS, bits, format, "s"] for assembly sources, where "name" is the name of the function without the module prefix and "class" is `rt` if the file is part of the libk runtime, or `fn` otherwise. this distinction is necessary because while the static library `libk.a` can include runtime objects, the shared library `libk.so` cannot. examples:

 * a C file in the module `kstr` named `kscomp` would be named `kstr/comp.fn.c`
 * a runtime assembly file called `boot` in the module `kcore` for x86-64 linux would be named `kcore/boot.rt.x86.lin.64.s`
 * the 32-bit x86 haiku version of a function called `kiowrite` defined in assembly would be named `kio/write.fn.x86.hai.32.s`.

each module should have a header named the same thing as the module except without the `k` prefix. (e.g. the header for `kio` is `kio/io.h`) located in its folder. this is the header that the end-user will be importing, and should handle any user-defined flags to present the API the user has selected.

each module should contain a markdown file. this file's name should be the name of the parent directory suffixed with `.md`; for instance, `kterm` should contain the file `kterm/kterm.md`. this file should document the module as thoroughly as possible 

each module may contain any number of files of the name `*.exe.c`. this files will be treated as *tools* by the build system and compiled as executables, rather than libraries. they should be compiled to `$to/$module.$tool`

the repository root and each module may also contain the directory `misc`. this directory may be used to store miscellaneous data such as ABI references, developer discussions, and roadmaps. if the `misc` directory is deleted, this must not affect the library or build system's function in any way - that is, nothing outside a `misc` folder may reference a `misc` folder or anything inside it, including documentation. the `misc` directory should be removed when its contents are no longer needed. in most cases, the repository wiki and forum should be used instead of the `misc` folder.

the folder `arch` in the root of the repository contains syscall tables and ABI implementations for various architectures.

# build system

................................................................................

 * `os={atom}` - an atom representing the operating system you are building libk for
 * `arch={atom}` - an atom representing the processor architecture you are building libk for
 * `bits={atom}` - if your processor has multiple variants with different word lengths (such as x86-32 vs. x86-64), specify the word length in this variable; otherwise, leave it unset.

further optional variables may be set to control the build process and determine what targets it produces.
 * `library=static {static|shared|both}` - this variable controls whether the build process will produce `libk.a`, `libk.so`, or both.
 * `to=out {path}` - an alternate path to store build artifacts in
 * `gen=gen {path}` - an alternate path to store generated source in
 * `cc=<autodetect> {executable}` - the compiler to compile C sources with
 * `m4=<autodetect> {executable}` - the m4 binary to compile the macro sources with
 * `asm=<autodetect> {executable}` - the assembler to assemble the assembly listings with. it must take Intel-syntax input and handle nasm-style macros. only `yasm` and `nasm` are likely to be viable.
 * `doc=yes {yes|no}` - whether to typeset the documentation (very slow with all three formats set to "yes")
 * `doc_html=yes {yes|no}` - enable or disable html output of the documentation
 * `doc_pdf=yes {yes|no}` - enable or disable pdf output of the documentation