examples.py (3872B)
1 import asyncio 2 import hashlib 3 import sys 4 from tempfile import TemporaryFile 5 6 sys.path += ".." 7 from make import hash_cache, rule, detach, Fetch, Task, Build, run_in_executor, shell 8 9 # Example rules 10 # Observe the general pattern that every rule is called to get a task, which can then be fetched. 11 # res = await fetch(rule(task_args...)) 12 13 14 def hash_file_sync(path: str): 15 h = hashlib.sha256() 16 with open(path, "rb") as f: 17 for chunk in f: 18 h.update(chunk) 19 return h.digest() 20 21 22 @rule 23 def hash_file(fetch: Fetch, path: str): 24 return run_in_executor(hash_file_sync, path) 25 26 27 @hash_cache 28 @rule 29 async def c_object( 30 fetch: Fetch, target: str, source: str | None = None, flags: str = "" 31 ): 32 if source is None: 33 if not target.endswith(".o"): 34 raise RuntimeError("Cannot infer source file") 35 source = target.removesuffix(".o") + ".c" 36 await fetch(hash_file(source)) 37 with TemporaryFile() as f: 38 await shell(f"gcc {flags} -MM -MT target {source} -c -o {target} -MF {f.name}") 39 # parse f 40 # new_deps = ("source.h",) 41 # await fetch.restart_if_out_of_date(new_deps) 42 43 44 @hash_cache 45 @rule 46 async def _eg_six(fetch: Fetch): 47 _ = fetch 48 six = 6 49 print(f"{six=}") 50 return six 51 52 53 @rule 54 async def _eg_thirtysix(fetch: Fetch): 55 # Here we await the dependencies serially. 56 # The second dependency cannot start until the first finishes. 57 six1 = await fetch(_eg_six()) 58 six2 = await fetch(_eg_six()) 59 print(f"{six1*six2=}") 60 return six1 * six2 61 62 63 @rule 64 async def _eg_multiply_add(fetch: Fetch, taskA: Task, taskB: Task, num: int): 65 # Here we await the dependencies in parallel. 66 a, b = await asyncio.gather(fetch(taskA), fetch(taskB)) 67 await asyncio.sleep(0.1) 68 print(f"{a*b+num=}") 69 return a * b + num 70 71 72 # When interfacing with inputs or in general anything outside the build system, 73 # Do NOT add @hash_cache, as it makes the task only rerun if a dependency was known to be modified. 74 # In this case, we have no real dependencies, and our output depends on the filesystem. 75 # So we leave out @hash_cache to ensure we always check that the file has not changed. 76 @rule 77 async def _eg_file(fetch: Fetch, filename: str): 78 _ = fetch 79 await asyncio.sleep(0.1) 80 with open(filename, "r") as f: 81 contents = f.readlines() 82 print("file", filename, "\n" + "".join(contents[1:5]), end="") 83 return contents 84 85 86 # Semaphores can be used to limit concurrency 87 _sem = asyncio.Semaphore(4) 88 89 90 @hash_cache 91 @rule 92 async def _eg_rec(fetch: Fetch, i: int): 93 if i // 3 - 1 >= 0: 94 # Instead of awaiting, dependencies can also be detached and run in the background. 95 detach(fetch(_eg_rec(i // 2 - 1))) 96 detach(fetch(_eg_rec(i // 3 - 1))) 97 else: 98 detach(fetch(_eg_file("make/__init__.py"))) 99 100 # Use semaphore to limit concurrency easily 101 async with _sem: 102 print("+ rec", i) 103 # Simulate some hard work 104 await asyncio.sleep(0.1) 105 print("- rec", i) 106 107 108 async def main(): 109 # To actually run the build system, 110 # 1) Create the store 111 # Use context manager to ensure the store is saved automatically when exiting 112 with Build(".makedb") as build: 113 # 2) Use it to await tasks 114 await build(_eg_rec(1234)) 115 await asyncio.gather( 116 build(_eg_thirtysix()), build(_eg_multiply_add(_eg_six(), _eg_six(), 6)) 117 ) 118 119 # Note that `build(...)` will wait for all detached jobs to complete before returning. 120 # You may instead use `build.build(...)`, which does not wait for detached jobs. 121 # You should ensure `detach.wait()` is called eventually so detached jobs can complete. 122 detach(build.build(_eg_rec(2345))) 123 await build.build(_eg_rec(3456)) 124 await detach.wait() 125 126 127 if __name__ == "__main__": 128 asyncio.run(main())