Use Jobserver With Ninja

cmake/cpack/appimage.cmake

@@ -1,3 +1,6 @@
+# Utility Functions
+include("${CMAKE_CURRENT_LIST_DIR}/../util/util.cmake")
+
 # Download AppImage Runtime
 set(RUNTIME_ARCH "unknown")
 if(CPACK_MCPI_ARCH STREQUAL "armhf")
@@ -8,18 +11,11 @@ elseif(CPACK_MCPI_ARCH STREQUAL "amd64")
     set(RUNTIME_ARCH "x86_64")
 endif()
 set(RUNTIME "${CPACK_TOPLEVEL_DIRECTORY}/runtime")
-file(DOWNLOAD
+safe_download(
+    "AppImage Runtime"
     "https://github.com/AppImage/type2-runtime/releases/download/continuous/runtime-${RUNTIME_ARCH}"
     "${RUNTIME}"
-    STATUS DOWNLOAD_STATUS
 )
-list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
-list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
-if(NOT STATUS_CODE EQUAL 0)
-    message(FATAL_ERROR "Unable To Download AppImage Runtime: ${ERROR_MESSAGE}")
-else()
-    message(STATUS "Downloaded AppImage Runtime: ${RUNTIME}")
-endif()
 
 # Package
 set(APPIMAGE_ARCH "unknown")

cmake/util/util.cmake

@@ -88,4 +88,20 @@ endfunction()
 function(set_and_mkdir name dir)
     set("${name}" "${dir}" PARENT_SCOPE)
     file(MAKE_DIRECTORY "${dir}")
+endfunction()
+
+# Download File With Error-Checking
+function(safe_download name url out)
+    file(DOWNLOAD
+        "${url}"
+        "${out}"
+        STATUS status
+    )
+    list(GET status 0 status_code)
+    list(GET status 1 error_message)
+    if(NOT status_code EQUAL 0)
+        message(FATAL_ERROR "Unable To Download ${name}: ${error_message}")
+    else()
+        message(STATUS "Downloaded ${name}: ${out}")
+    endif()
 endfunction()

scripts/build.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import * as path from 'node:path';
 import * as fs from 'node:fs';
-import { info, err, run, createDir, getScriptsDir } from './lib/util.mjs';
+import { info, err, run, createDir, getScriptsDir, getBuildToolsBin, getParallelFlag } from './lib/util.mjs';
 import { parseOptions, Enum, Architectures } from './lib/options.mjs';
 
 // CMake Options
@@ -98,6 +98,20 @@ if (!options.install) {
     createDir(out, !useOutRoot);
 }
 
+// Use Build Tools
+const buildTools = getBuildToolsBin();
+const hasBuildTools = fs.existsSync(buildTools);
+if (hasBuildTools) {
+    function prependEnv(env, value) {
+        const old = process.env[env];
+        if (old) {
+            value += ':' + old;
+        }
+        process.env[env] = value;
+    }
+    prependEnv('PATH', getBuildToolsBin());
+}
+
 // Run CMake
 const configure = ['cmake', '-GNinja Multi-Config'];
 cmakeOptions.forEach((value, key) => {
@@ -107,8 +121,14 @@ configure.push('-S', root, '-B', build);
 run(configure);
 
 // Build
-const configArg = ['--config', options.debug ? 'Debug' : 'Release'];
-run(['cmake', '--build', build, ...configArg]);
+const config = options.debug ? 'Debug' : 'Release';
+const configArg = ['--config', config];
+if (hasBuildTools) {
+    fs.writeFileSync(path.join(build, 'Makefile'), `.PHONY: all\nall:\n\t+@ninja -f build-${config}.ninja\n`);
+    run(['make', '-C', build, getParallelFlag(), '--jobserver-style=fifo']);
+} else {
+    run(['cmake', '--build', build, getParallelFlag(), ...configArg]);
+}
 
 // Package
 if (options.packageType !== PackageTypes.AppImage) {

scripts/fix-appimage-for-docker.sh

@@ -1,2 +1,7 @@
 #!/bin/sh
-exec sed -i '0,/AI\x02/{s|AI\x02|\x00\x00\x00|}' "$1"
+
+set -e
+
+FILE="$1"
+chmod +x "${FILE}"
+sed -i '0,/AI\x02/{s|AI\x02|\x00\x00\x00|}' "${FILE}"

scripts/install-dependencies.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import * as path from 'node:path';
 import * as fs from 'node:fs';
-import { err, run, makeExecutable, getDebianVersion, getScriptsDir, info, doesPackageExist } from './lib/util.mjs';
+import { err, run, getDebianVersion, info, doesPackageExist, getBuildToolsDir, createDir, getBuildToolsBin, getParallelFlag } from './lib/util.mjs';
 import { parseOptions, Enum, Architectures } from './lib/options.mjs';
 
 // Check System
@@ -47,18 +47,26 @@ run(['apt-get', 'dist-upgrade', '-y']);
 // Install Packages
 const packages = [];
 function addPackageForBuild(...arr) {
-    // The machine the package is built on.
+    // This will install packages thet match
+    // the build machine's architecture.
+    // This is usually used for build tools.
     packages.push(...arr);
 }
 function getPackageForHost(name) {
     return name + archSuffix;
 }
 function addPackageForHost(...arr) {
-    // The machine the package is built for.
+    // This will install packages thet match
+    // the host machine's architecture.
+    // This is usually used for libraries.
     for (const name of arr) {
         packages.push(getPackageForHost(name));
     }
 }
+function installPackages() {
+    // Install Queued Packages
+    run(['apt-get', 'install', '--no-install-recommends', '-y', ...packages]);
+}
 
 // Build Dependencies
 const handlers = new Map();
@@ -67,7 +75,9 @@ handlers.set(Modes.Build, function () {
     addPackageForBuild(
         'git',
         'cmake' + backportsSuffix,
-        'ninja-build'
+        // For Build Tools
+        'make',
+        're2c'
     );
 
     // Compiler
@@ -107,23 +117,21 @@ handlers.set(Modes.Build, function () {
         'zsync'
     );
 
-    // Download AppImageTool
-    function getAppImageArch() {
-        switch (process.arch) {
-            case 'x64': return 'x86_64';
-            case 'arm64': return 'aarch64';
-            case 'arm': return 'armhf';
-            default: err('Unsupported Build Architecture');
-        }
-    }
-    let appimagetool = '/opt/appimagetool';
-    run(['wget', '-O', appimagetool, `https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-${getAppImageArch()}.AppImage`]);
-    makeExecutable(appimagetool);
-    run([getScriptsDir() + '/fix-appimage-for-docker.sh', appimagetool]);
-    const script = `#!/bin/sh\nexec ${appimagetool} --appimage-extract-and-run "$@"\n`;
-    appimagetool = '/usr/local/bin/' + path.basename(appimagetool);
-    fs.writeFileSync(appimagetool, script);
-    makeExecutable(appimagetool);
+    // Install Packages
+    installPackages();
+
+    // Build Tools
+    const buildToolsDir = getBuildToolsDir();
+    const buildDir = path.join(buildToolsDir, 'build');
+    createDir(buildDir, false);
+    run([
+        'cmake',
+        '-DCMAKE_BUILD_TYPE=Release',
+        '-DCMAKE_INSTALL_PREFIX=' + getBuildToolsBin(),
+        '-S', buildToolsDir, '-B', buildDir
+    ]);
+    run(['cmake', '--build', buildDir, getParallelFlag()]);
+    run(['cmake', '--install', buildDir]);
 });
 
 // Testing Dependencies
@@ -139,6 +147,7 @@ handlers.set(Modes.Test, function () {
         'libopenal1',
         glib
     );
+    installPackages();
 });
 
 // SDK Usage Dependencies
@@ -149,10 +158,8 @@ handlers.set(Modes.SDK, function () {
         'g++-arm-linux-gnueabihf',
         'gcc-arm-linux-gnueabihf'
     );
+    installPackages();
 });
 
 // Run
-handlers.get(options.mode)();
-
-// Install Packages
-run(['apt-get', 'install', '--no-install-recommends', '-y', ...packages]);
+handlers.get(options.mode)();

scripts/lib/util.mjs

@@ -2,6 +2,7 @@ import * as child_process from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import * as url from 'node:url';
+import * as os from 'node:os';
 
 // Logging
 const EXIT_FAILURE = 1;
@@ -56,22 +57,24 @@ export function getDebianVersion() {
     return 'unknown';
 }
 
-// Make File Executable
-export function makeExecutable(path) {
-    fs.chmodSync(path,
-        fs.constants.S_IRUSR |
-        fs.constants.S_IWUSR |
-        fs.constants.S_IXUSR |
-        fs.constants.S_IRGRP |
-        fs.constants.S_IXGRP |
-        fs.constants.S_IROTH |
-        fs.constants.S_IXOTH
-    );
-}
-
 // Get Scripts Directory
 export function getScriptsDir() {
     const __filename = url.fileURLToPath(import.meta.url);
     const __dirname = path.dirname(__filename);
     return path.join(__dirname, '..');
+}
+
+// Custom Build Tools
+export function getBuildToolsDir() {
+    return path.join(getScriptsDir(), '..', 'tools');
+}
+export function getBuildToolsBin() {
+    const dir = path.join(getBuildToolsDir(), 'bin');
+    createDir(dir, false);
+    return dir;
+}
+
+// Get '-jX' Flag For Build Tools
+export function getParallelFlag() {
+    return '-j' + os.cpus().length;
 }

tools/.gitignore

@@ -0,0 +1,2 @@
+/build
+/bin

tools/CMakeLists.txt

@@ -0,0 +1,61 @@
+cmake_minimum_required(VERSION 3.25.0)
+project(tools)
+
+# Utility Functions
+include(../cmake/util/util.cmake)
+
+# Options
+force_set(BUILD_TESTING FALSE BOOL)
+force_set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/bin" FILEPATH)
+force_set(CMAKE_INSTALL_BINDIR "." FILEPATH)
+
+# Ninja
+include(FetchContent)
+FetchContent_Declare(ninja
+    GIT_REPOSITORY "https://github.com/digit-google/ninja.git"
+    GIT_TAG jobserver
+)
+FetchContent_MakeAvailable(ninja)
+
+# Make
+include(ExternalProject)
+ExternalProject_Add(make
+    URL "https://ftp.gnu.org/gnu/make/make-4.4.1.tar.gz"
+    URL_HASH "SHA256=dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3"
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    CONFIGURE_COMMAND
+        "<SOURCE_DIR>/configure"
+        "--prefix=<INSTALL_DIR>"
+        "--without-guile"
+        "--enable-silent-rules"
+        "--quiet"
+        "CFLAGS=${CMAKE_C_FLAGS}"
+    BUILD_IN_SOURCE TRUE
+    INSTALL_COMMAND ""
+)
+ExternalProject_Get_Property(make SOURCE_DIR)
+install(PROGRAMS "${SOURCE_DIR}/make" DESTINATION "${CMAKE_INSTALL_BINDIR}")
+
+# Download AppImageTool
+execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(ARCH STREQUAL "armv8b" OR ARCH STREQUAL "armv8l")
+    set(ARCH "aarch64")
+endif()
+set(APPIMAGETOOL_NAME "appimagetool")
+set(APPIMAGETOOL_BIN "${CMAKE_CURRENT_BINARY_DIR}/${APPIMAGETOOL_NAME}.AppImage")
+safe_download(
+    "AppImageTool"
+    "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-${ARCH}.AppImage"
+    "${APPIMAGETOOL_BIN}"
+)
+# Fix
+execute_process(
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/../scripts/fix-appimage-for-docker.sh" "${APPIMAGETOOL_BIN}"
+    COMMAND_ERROR_IS_FATAL ANY
+)
+# Install
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${APPIMAGETOOL_NAME}"
+    "#!/bin/sh\n"
+    "exec \"${APPIMAGETOOL_BIN}\" --appimage-extract-and-run \"\$@\"\n"
+)
+install(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/${APPIMAGETOOL_NAME}" DESTINATION "${CMAKE_INSTALL_BINDIR}")