#!/usr/bin/env python3
import os
import subprocess
from pathlib import Path
# NOTE: Do NOT import anything here that needs be built (e.g. params)
from openpilot.common.basedir import BASEDIR
from openpilot.common.spinner import Spinner
from openpilot.common.text_window import TextWindow
from openpilot.system.hardware import AGNOS
from openpilot.common.swaglog import cloudlog, add_file_handler
from openpilot.system.version import get_build_metadata
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
TOTAL_SCONS_NODES = 2560
MAX_BUILD_PROGRESS = 100
[docs]
def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
nproc = os.cpu_count()
if nproc is None:
nproc = 2
extra_args = ["--minimal"] if minimal else []
# building with all cores can result in using too
# much memory, so retry with less parallelism
compile_output: list[bytes] = []
for n in (nproc, nproc/2, 1):
compile_output.clear()
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
assert scons.stderr is not None
# Read progress from stderr and update spinner
while scons.poll() is None:
try:
line = scons.stderr.readline()
if line is None:
continue
line = line.rstrip()
prefix = b'progress: '
if line.startswith(prefix):
i = int(line[len(prefix):])
spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
elif len(line):
compile_output.append(line)
print(line.decode('utf8', 'replace'))
except Exception:
pass
if scons.returncode == 0:
break
if scons.returncode != 0:
# Read remaining output
if scons.stderr is not None:
compile_output += scons.stderr.read().split(b'\n')
# Build failed log errors
error_s = b"\n".join(compile_output).decode('utf8', 'replace')
add_file_handler(cloudlog)
cloudlog.error("scons build failed\n" + error_s)
# Show TextWindow
spinner.close()
if not os.getenv("CI"):
with TextWindow("openpilot failed to build\n \n" + error_s) as t:
t.wait_for_exit()
exit(1)
# enforce max cache size
cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
cache_files.sort(key=lambda f: f.stat().st_mtime)
cache_size = sum(f.stat().st_size for f in cache_files)
for f in cache_files:
if cache_size < MAX_CACHE_SIZE:
break
cache_size -= f.stat().st_size
f.unlink()
if __name__ == "__main__":
spinner = Spinner()
spinner.update_progress(0, 100)
build_metadata = get_build_metadata()
build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)