Coverage for jetgp/setup_otilib.py: 11%
157 statements
« prev ^ index » next coverage.py v7.10.7, created at 2026-05-07 16:12 -0500
1#!/usr/bin/env python3
2"""
3setup_otilib.py — Patch otilib-master with JetGP's required modifications
4and optionally run the full build workflow.
6Usage:
7 python setup_otilib.py # interactive
8 python setup_otilib.py --otilib /path/to/otilib-master
9 python setup_otilib.py --otilib /path/to/otilib-master --build
10 python setup_otilib.py --otilib /path/to/otilib-master --build --workers 8
11"""
13import argparse
14import os
15import re
16import shutil
17import site
18import subprocess
19import sys
20from pathlib import Path
22# ---------------------------------------------------------------------------
23# Config file — persists the otilib path for get_oti_module() auto-detection
24# ---------------------------------------------------------------------------
25JETGP_CONFIG_FILE = Path.home() / ".config" / "jetgp" / "otilib_path"
27# ---------------------------------------------------------------------------
28# File map: list of (source relative to otilib_mods/, destination relative to otilib root)
29# A source file may appear more than once to copy it to multiple destinations.
30# ---------------------------------------------------------------------------
31FILE_MAP = [
32 ("src_CMakeLists.txt", "src/CMakeLists.txt"),
33 ("src_python_pyoti_CMakeLists.txt", "src/python/pyoti/CMakeLists.txt"),
34 ("regenerate_all_c.py", "build/regenerate_all_c.py"),
35 ("build_static.py", "build/build_static.py"),
36 ("cmod_writer.py", "build/pyoti/cmod_writer.py"),
37 ("cmod_writer.py", "src/python/pyoti/python/cmod_writer.py"),
38 ("creators.pxi",
39 "src/python/pyoti/python/source_conv/"
40 "src/python/pyoti/cython/static/number/creators.pxi"),
41 ("include.pxi",
42 "src/python/pyoti/python/source_conv/"
43 "src/python/pyoti/cython/static/number/include.pxi"),
44 ("array_base.pxi",
45 "src/python/pyoti/python/source_conv/"
46 "src/python/pyoti/cython/static/number/array/base.pxi"),
47 ("rebuild_all_static.py", "build/rebuild_all_static.py"),
48 ("rebuild_all_static.sh", "build/rebuild_all_static.sh"),
49]
51# ---------------------------------------------------------------------------
52# Build scripts whose hardcoded paths need to be rewritten
53# ---------------------------------------------------------------------------
54SCRIPT_PATCHES = [
55 (
56 "CMakeLists.txt",
57 [
58 (
59 r'set\(CMAKE_C_FLAGS_RELEASE\s+"-O3 -Wall -std=c99"\)',
60 'set(CMAKE_C_FLAGS_RELEASE "-O1 -Wall -std=c99")',
61 ),
62 ],
63 ),
64 (
65 "src/python/pyoti/cython/setup.py.in",
66 [
67 (
68 r'-O3',
69 '-O1',
70 ),
71 ],
72 ),
73 (
74 "build/regenerate_all_c.py",
75 [
76 (
77 r'BASE_DIR\s*=\s*"[^"]+"',
78 'BASE_DIR = "{otilib}"',
79 ),
80 (
81 r'sys\.path\.insert\(0,\s*"[^"]+"\)',
82 'sys.path.insert(0, "{otilib}/src/python/pyoti/python")',
83 ),
84 (
85 r'(print\(f?" 1\. cd )[^"]+(")',
86 r'\g<1>{otilib}/build\g<2>',
87 ),
88 ],
89 ),
90 (
91 "build/build_static.py",
92 [
93 (
94 r'PROJECT_ROOT\s*=\s*"[^"]+"',
95 'PROJECT_ROOT = "{otilib}"',
96 ),
97 ],
98 ),
99 (
100 "build/rebuild_all_static.py",
101 [
102 (
103 r'BUILD_DIR\s*=\s*"[^"]+"',
104 'BUILD_DIR = "{otilib}/build"',
105 ),
106 (
107 r'PYTHON\s*=\s*"[^"]+"',
108 'PYTHON = "{python}"',
109 ),
110 ],
111 ),
112 (
113 "build/rebuild_all_static.sh",
114 [
115 (
116 r'ls [^\s]+/src/c/static/onumm\*\.c',
117 'ls {otilib}/src/c/static/onumm*.c',
118 ),
119 ],
120 ),
121]
124def resolve_otilib(path_arg):
125 """Resolve and validate the otilib root directory."""
126 p = Path(path_arg).expanduser().resolve()
127 if not p.is_dir():
128 sys.exit(f"Error: otilib path does not exist: {p}")
129 if not (p / "src" / "CMakeLists.txt").exists():
130 sys.exit(
131 f"Error: {p} does not look like an otilib-master root "
132 "(missing src/CMakeLists.txt)"
133 )
134 return p
137def copy_mod_files(mods_dir: Path, otilib: Path):
138 print("\n[1/3] Copying mod files to otilib-master...")
139 for src_name, dst_rel in FILE_MAP:
140 src = mods_dir / src_name
141 dst = otilib / dst_rel
142 if not src.exists():
143 sys.exit(f"Error: mod file not found: {src}")
144 dst.parent.mkdir(parents=True, exist_ok=True)
145 shutil.copy2(src, dst)
146 print(f" {src_name} -> {dst_rel}")
147 print(" Done.")
150def patch_build_scripts(otilib: Path, python_exe: str):
151 print("\n[2/3] Patching hardcoded paths in build scripts...")
152 for rel_path, patches in SCRIPT_PATCHES:
153 script = otilib / rel_path
154 if not script.exists():
155 print(f" WARNING: {rel_path} not found, skipping.")
156 continue
157 text = script.read_text()
158 original = text
159 for pattern, template in patches:
160 replacement = template.format(otilib=str(otilib), python=python_exe)
161 text = re.sub(pattern, replacement, text)
162 if text != original:
163 script.write_text(text)
164 print(f" Patched: {rel_path}")
165 else:
166 print(f" Already up to date: {rel_path}")
167 print(" Done.")
170def save_config(otilib: Path):
171 """Write otilib path to ~/.config/jetgp/otilib_path."""
172 JETGP_CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
173 JETGP_CONFIG_FILE.write_text(str(otilib))
174 print(f" Saved otilib path to {JETGP_CONFIG_FILE}")
177def install_pyoti_to_path(otilib: Path):
178 """
179 Make pyoti importable in the active environment by writing a .pth file
180 pointing to otilib-master/build/ into site-packages.
181 This replicates what `conda develop .` does from the build directory.
182 """
183 build_dir = str(otilib / "build")
184 site_packages = site.getsitepackages()[0]
185 pth_file = Path(site_packages) / "otilib.pth"
186 pth_file.write_text(build_dir + "\n")
187 print(f" Wrote {pth_file} -> {build_dir}")
190def run_build(otilib: Path, workers: int):
191 build_dir = otilib / "build"
192 python = sys.executable
194 # Ensure cmake uses the conda environment's compilers, not the system ones.
195 conda_prefix = os.environ.get("CONDA_PREFIX")
196 if conda_prefix:
197 conda_bin = Path(conda_prefix) / "bin"
198 if (conda_bin / "gcc").exists():
199 os.environ["CC"] = str(conda_bin / "gcc")
200 print(f" Set CC={os.environ['CC']}")
201 if (conda_bin / "gfortran").exists():
202 os.environ["FC"] = str(conda_bin / "gfortran")
203 print(f" Set FC={os.environ['FC']}")
205 # Clean up shipped static module files not needed by JetGP before
206 # the bootstrap build (avoids Cythonizing unwanted .pxd headers).
207 # Parse ALL_MODULES from the deployed regenerate_all_c.py.
208 print("\n >> Cleaning up unwanted shipped static modules...")
209 regen_text = (build_dir / "regenerate_all_c.py").read_text()
210 match = re.search(r'ALL_MODULES\s*=\s*\[([^\]]+)\]', regen_text)
211 wanted = set()
212 if match:
213 for m, n in re.findall(r'\((\d+)\s*,\s*(\d+)\)', match.group(1)):
214 wanted.add(f"onumm{m}n{n}")
215 clean_dirs = [
216 otilib / "src" / "c" / "static",
217 otilib / "src" / "python" / "pyoti" / "cython" / "static",
218 otilib / "include" / "pyoti" / "static",
219 ]
220 for d in clean_dirs:
221 for ext in (".c", ".pyx", ".pxd"):
222 for f in d.glob(f"onumm*{ext}"):
223 name = f.stem
224 if name not in wanted:
225 f.unlink()
226 companion = f.with_suffix("")
227 if companion.is_dir():
228 shutil.rmtree(companion)
229 print(f" Removed: {f.relative_to(otilib)}")
231 # Write an empty static.c for the bootstrap — no static modules are
232 # needed yet, and the wanted .c files may not exist until after
233 # regenerate_all_c.py runs. The second cmake+make will pick up the
234 # regenerated sources via the individual mXnY targets.
235 static_c = otilib / "src" / "c" / "static.c"
236 if static_c.exists():
237 static_c.write_text("/* placeholder — static modules built as individual targets */\n")
238 print(" Wrote empty static.c (static modules built as individual targets)")
240 # Disable examples build — they link against static module symbols
241 # that are no longer compiled into the oti library.
242 top_cmake = otilib / "CMakeLists.txt"
243 if top_cmake.exists():
244 lines = top_cmake.read_text().splitlines(True)
245 with open(top_cmake, 'w') as f:
246 for line in lines:
247 if 'add_subdirectory' in line and 'examples' in line:
248 f.write('# ' + line)
249 print(f" Commented out: {line.strip()}")
250 else:
251 f.write(line)
252 print(" Disabled examples build in top-level CMakeLists.txt")
254 # Bootstrap: build pyoti.core first so that regenerate_all_c.py can
255 # import cmod_writer (which depends on pyoti.core).
256 bootstrap_steps = [
257 (
258 "Running cmake .. (bootstrap)",
259 ["cmake", ".."],
260 str(build_dir),
261 ),
262 (
263 f"Running make oticython -j{workers} (bootstrap — builds pyoti.core only)",
264 ["make", "oticython", f"-j{workers}"],
265 str(build_dir),
266 ),
267 (
268 "Running make gendata (generates data files needed by cmod_writer)",
269 ["make", "gendata"],
270 str(build_dir),
271 ),
272 ]
274 print("\n[3/3] Running build workflow...")
275 for description, cmd, cwd in bootstrap_steps:
276 print(f"\n >> {description}")
277 print(f" {' '.join(cmd)} (cwd: {cwd})")
278 result = subprocess.run(cmd, cwd=cwd)
279 if result.returncode != 0:
280 sys.exit(f"\nError: step failed: {description}")
282 print(f"\n >> Making pyoti importable in active environment...")
283 install_pyoti_to_path(otilib)
285 # Now that pyoti.core is available, regenerate sources and rebuild.
286 steps = [
287 (
288 "Regenerating C/Cython sources from templates",
289 [python, "regenerate_all_c.py"],
290 str(build_dir),
291 ),
292 (
293 "Running cmake .. (picks up regenerated sources)",
294 ["cmake", ".."],
295 str(build_dir),
296 ),
297 ]
299 # Build only the individual mXnY static library targets (the core
300 # libs are already built from the bootstrap). This avoids a full
301 # parallel make that triggers Fortran .mod file race conditions.
302 if wanted:
303 make_targets = [f"m{name[5:]}" for name in sorted(wanted)] # onummXnY -> mXnY
304 steps.append((
305 f"Building static library targets: {' '.join(make_targets)}",
306 ["make"] + make_targets + [f"-j{workers}"],
307 str(build_dir),
308 ))
309 else:
310 steps.append((
311 f"Running make -j{workers}",
312 ["make", f"-j{workers}"],
313 str(build_dir),
314 ))
316 for description, cmd, cwd in steps:
317 print(f"\n >> {description}")
318 print(f" {' '.join(cmd)} (cwd: {cwd})")
319 result = subprocess.run(cmd, cwd=cwd)
320 if result.returncode != 0:
321 sys.exit(f"\nError: step failed: {description}")
323 print(f"\n >> Building all Cython static modules ({workers} workers)...")
324 result = subprocess.run(
325 ["bash", "rebuild_all_static.sh", str(workers)],
326 cwd=str(build_dir),
327 )
328 if result.returncode != 0:
329 sys.exit("\nError: static module build failed.")
331 print("\n Build complete. Static modules are in otilib-master/build/pyoti/static/")
332 print(" pyoti is importable via the .pth file written to site-packages.")
335def main():
336 parser = argparse.ArgumentParser(
337 description="Patch otilib-master with JetGP modifications and optionally build."
338 )
339 parser.add_argument(
340 "--otilib",
341 metavar="PATH",
342 help="Path to your otilib-master directory",
343 )
344 parser.add_argument(
345 "--build",
346 action="store_true",
347 help="Run the full build workflow after patching",
348 )
349 parser.add_argument(
350 "--workers",
351 type=int,
352 default=4,
353 help="Parallel workers for the Cython build step (default: 4)",
354 )
355 args = parser.parse_args()
357 # Locate otilib_mods/ relative to this script (repo root, one level up)
358 mods_dir = Path(__file__).parent.parent / "otilib_mods"
359 if not mods_dir.is_dir():
360 sys.exit(f"Error: otilib_mods/ not found next to this script ({mods_dir})")
362 # Resolve otilib path
363 if args.otilib:
364 otilib = resolve_otilib(args.otilib)
365 elif JETGP_CONFIG_FILE.exists():
366 candidate = JETGP_CONFIG_FILE.read_text().strip()
367 if Path(candidate).is_dir():
368 print(f"Using otilib path from config: {candidate}")
369 otilib = Path(candidate)
370 else:
371 sys.exit(
372 f"Error: saved otilib path '{candidate}' no longer exists.\n"
373 f"Run again with --otilib /path/to/otilib-master"
374 )
375 else:
376 default = Path.home() / "otilib-master"
377 prompt = f"Path to otilib-master [{default}]: "
378 answer = input(prompt).strip()
379 otilib = resolve_otilib(answer if answer else default)
381 python_exe = sys.executable
382 print(f"\notilib root : {otilib}")
383 print(f"Python : {python_exe}")
385 copy_mod_files(mods_dir, otilib)
386 patch_build_scripts(otilib, python_exe)
387 save_config(otilib)
389 if args.build:
390 run_build(otilib, args.workers)
391 else:
392 print(
393 "\nPatching complete. To build, run:\n"
394 f" python -m jetgp.setup_otilib --otilib {otilib} --build --workers 8"
395 )
398if __name__ == "__main__":
399 main()