commit e6a64f5972b99b6fa6a8bf0f0a879449fbde9bd6
parent 09ea2700c79218ee15fb8263f0fdf65657619aa4
Author: gracefu <81774659+gracefuu@users.noreply.github.com>
Date: Tue, 15 Apr 2025 05:09:27 +0800
Move examples to own file
Diffstat:
| M | .gitignore | | | 1 | + |
| A | examples.py | | | 93 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | make.py | | | 122 | +++++++++++++++---------------------------------------------------------------- |
3 files changed, 117 insertions(+), 99 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
make.db
+__pycache__
diff --git a/examples.py b/examples.py
@@ -0,0 +1,93 @@
+from make import cache, rule, detach, Fetch, Task, Build
+import asyncio
+
+# Example rules
+# Observe the general pattern that every rule is called to get a task, which can then be fetched.
+# res = await fetch(rule(task_args...))
+
+
+@cache
+@rule
+async def _eg_six(fetch: Fetch):
+ _ = fetch
+ six = 6
+ print(f"{six=}")
+ return six
+
+
+@rule
+async def _eg_thirtysix(fetch: Fetch):
+ # Here we await the dependencies serially.
+ # The second dependency cannot start until the first finishes.
+ six1 = await fetch(_eg_six())
+ six2 = await fetch(_eg_six())
+ print(f"{six1*six2=}")
+ return six1 * six2
+
+
+@rule
+async def _eg_multiply_add(fetch: Fetch, taskA: Task, taskB: Task, num: int):
+ # Here we await the dependencies in parallel.
+ a, b = await asyncio.gather(fetch(taskA), fetch(taskB))
+ await asyncio.sleep(0.1)
+ print(f"{a*b+num=}")
+ return a * b + num
+
+
+# When interfacing with inputs or in general anything outside the build system,
+# Do NOT add @ctRebuilder, as it makes the task only rerun if a dependency was known to be modified.
+# In this case, we have no real dependencies, and our output depends on the filesystem.
+# So we leave out @ctRebuilder to ensure we always check that the file has not changed.
+@rule
+async def _eg_file(fetch: Fetch, filename: str):
+ _ = fetch
+ await asyncio.sleep(0.1)
+ with open(filename, "r") as f:
+ contents = f.readlines()
+ print("file", filename, "\n" + "".join(contents[1:5]), end="")
+ return contents
+
+
+# Semaphores can be used to limit concurrency
+_sem = asyncio.Semaphore(4)
+
+
+@cache
+@rule
+async def _eg_rec(fetch: Fetch, i: int):
+ if i // 3 - 1 >= 0:
+ # Instead of awaiting, dependencies can also be detached and run in the background.
+ detach(fetch(_eg_rec(i // 2 - 1)))
+ detach(fetch(_eg_rec(i // 3 - 1)))
+ else:
+ detach(fetch(_eg_file("make.py")))
+
+ # Use semaphore to limit concurrency easily
+ async with _sem:
+ print("+ rec", i)
+ # Simulate some hard work
+ await asyncio.sleep(0.1)
+ print("- rec", i)
+
+
+async def run_examples():
+ # To actually run the build system,
+ # 1) Create the store
+ # Use context manager to ensure the store is saved automatically when exiting
+ with Build("make.db") as build:
+ # 2) Use it to await tasks
+ await build(_eg_rec(1234))
+ await asyncio.gather(
+ build(_eg_thirtysix()), build(_eg_multiply_add(_eg_six(), _eg_six(), 6))
+ )
+
+ # Note that `build(...)` will wait for all detached jobs to complete before returning.
+ # You may choose to use the lower level `build.fetch(...)` function instead, which does not wait for detached jobs.
+ # You must then ensure `build.wait()` is called later to wait for detached jobs to complete.
+ await build.fetch(_eg_rec(2345))
+ await build.fetch(_eg_rec(3456))
+ await build.wait()
+
+
+if __name__ == "__main__":
+ asyncio.run(run_examples())
diff --git a/make.py b/make.py
@@ -34,7 +34,7 @@ import hashlib
from typing import Awaitable, Callable, Any, Concatenate, Optional
-FetchFn = Callable[["Task"], Awaitable[Any]]
+Fetch = Callable[["Task"], Awaitable[Any]]
TaskKey = str
RuleKey = str
@@ -60,7 +60,7 @@ def _fn_to_key(fn) -> str:
class Task:
task_key: TaskKey
- rule_fn: Callable[Concatenate[FetchFn, TaskKey, "Store", ...], Awaitable[Any]]
+ rule_fn: Callable[Concatenate[Fetch, TaskKey, "Store", ...], Awaitable[Any]]
args: tuple
hash: int
@@ -81,7 +81,7 @@ class Task:
self.args = args
self.hash = hash(self.task_key)
- def __call__(self, fetch: "FetchFn", store: "Store"):
+ def __call__(self, fetch: "Fetch", store: "Store"):
return self.rule_fn(fetch, self.task_key, store, *self.args)
def __repr__(self):
@@ -96,7 +96,7 @@ class Task:
class Rule:
rule_key: RuleKey
- rule_fn: Callable[Concatenate[FetchFn, TaskKey, "Store", ...], Awaitable[Any]]
+ rule_fn: Callable[Concatenate[Fetch, TaskKey, "Store", ...], Awaitable[Any]]
hash: int
@staticmethod
@@ -153,7 +153,7 @@ class Rules:
def cache(self):
def decorator(rule: Rule):
@functools.wraps(rule.rule_fn)
- async def new_rule_fn(fetch: FetchFn, task_key: str, store: "Store", *args):
+ async def new_rule_fn(fetch: Fetch, task_key: str, store: "Store", *args):
past_runs = store.key_info[task_key]
output_value = store.key_value[task_key]
possible_values = []
@@ -208,7 +208,6 @@ class Store:
self.filename = filename
self.rules = rules
- self.mutex = asyncio.Semaphore()
self.key_value = collections.defaultdict(_fNone)
self.key_info = collections.defaultdict(list)
@@ -244,10 +243,6 @@ class SuspendingFetch:
self.done = dict()
self.waits = dict()
- async def __call__(self, task: Task):
- await self.fetch(task)
- await self.wait()
-
async def wait(self):
while _background_tasks:
await asyncio.gather(*_background_tasks)
@@ -273,95 +268,24 @@ class SuspendingFetch:
return result
-# Example rules
-# Observe the general pattern that every rule is called to get a task, which can then be fetched.
-# res = await fetch(rule(task_args...))
-
-
-@cache
-@rule
-async def _eg_six(fetch: FetchFn):
- _ = fetch
- six = 6
- print(f"{six=}")
- return six
-
-
-@rule
-async def _eg_thirtysix(fetch: FetchFn):
- # Here we await the dependencies serially.
- # The second dependency cannot start until the first finishes.
- six1 = await fetch(_eg_six())
- six2 = await fetch(_eg_six())
- print(f"{six1*six2=}")
- return six1 * six2
-
-
-@rule
-async def _eg_multiply_add(fetch: FetchFn, taskA: Task, taskB: Task, num: int):
- # Here we await the dependencies in parallel.
- a, b = await asyncio.gather(fetch(taskA), fetch(taskB))
- await asyncio.sleep(0.1)
- print(f"{a*b+num=}")
- return a * b + num
-
-
-# When interfacing with inputs or in general anything outside the build system,
-# Do NOT add @ctRebuilder, as it makes the task only rerun if a dependency was known to be modified.
-# In this case, we have no real dependencies, and our output depends on the filesystem.
-# So we leave out @ctRebuilder to ensure we always check that the file has not changed.
-@rule
-async def _eg_file(fetch: FetchFn, filename: str):
- _ = fetch
- await asyncio.sleep(0.1)
- with open(filename, "r") as f:
- contents = f.readlines()
- print("file", filename, "\n" + "".join(contents[1:5]), end="")
- return contents
-
-
-# Semaphores can be used to limit concurrency
-_sem = asyncio.Semaphore(4)
-
-
-@cache
-@rule
-async def _eg_rec(fetch: FetchFn, i: int):
- if i // 3 - 1 >= 0:
- # Instead of awaiting, dependencies can also be detached and run in the background.
- detach(fetch(_eg_rec(i // 2 - 1)))
- detach(fetch(_eg_rec(i // 3 - 1)))
- else:
- detach(fetch(_eg_file("make.py")))
-
- # Use semaphore to limit concurrency easily
- async with _sem:
- print("+ rec", i)
- # Simulate some hard work
- await asyncio.sleep(0.1)
- print("- rec", i)
-
-
-async def run_examples():
- # To actually run the build system,
- # 1) Create the store
- # Use context manager to ensure the store is saved automatically when exiting
- with Store("make.db", _rules) as store:
- # 2) Create the fetch callable
- fetch = SuspendingFetch(store)
- # 3) Use it to await tasks
- await fetch(_eg_rec(1234))
- await asyncio.gather(
- fetch(_eg_thirtysix()), fetch(_eg_multiply_add(_eg_six(), _eg_six(), 6))
- )
+class Build:
+ def __init__(self, filename, rules=_rules):
+ self._store = Store(filename, rules)
+ self._fetch = SuspendingFetch(self._store)
+
+ async def __call__(self, task: Task):
+ await self.fetch(task)
+ await self.wait()
- # Note that `fetch(...)` will wait for all detached jobs to complete before returning.
- # You may choose to use the lower level `fetch.fetch(...)` function instead, which does not wait for detached jobs.
- # You must then ensure `fetch.wait()` is called later to wait for detached jobs to complete.
- await fetch.fetch(_eg_rec(2345))
- await fetch.fetch(_eg_rec(3456))
- await fetch.wait()
+ def wait(self):
+ return self._fetch.wait()
+ def fetch(self, task: Task):
+ return self._fetch.fetch(task)
-if __name__ == "__main__":
- asyncio.run(run_examples())
+ def __enter__(self):
+ self._store.__enter__()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self._store.__exit__(exc_type, exc_val, exc_tb)