commit c23c0c74c132b7369c298b5270c8d383b94cc212
parent 85c3f1e41b1125583a213ca5802eff436a058a88
Author: gracefu <81774659+gracefuu@users.noreply.github.com>
Date: Thu, 24 Apr 2025 08:47:59 +0800
Test using http.client instead of shell+curl
Diffstat:
| M | make.py | | | 100 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------- |
| M | make3/helpers.py | | | 50 | ++++++++++++++++++++++++++++++-------------------- |
2 files changed, 118 insertions(+), 32 deletions(-)
diff --git a/make.py b/make.py
@@ -2,6 +2,7 @@ from make3 import EchoAll, file_hash, make_main, once, shell
import asyncio
import json
import os
+import http.client
async def main():
@@ -14,10 +15,21 @@ async def main():
APIPASSWORD = (await shell("secret-tool lookup owner grace@bunny.net")).utf8stdout
LOCALPATH = "public"
- STORAGEURL = f"https://sg.storage.bunnycdn.com/{STORAGENAME}"
- STORAGECMD = f"curl -H 'AccessKey: {STORAGEPASSWORD}' -s"
- PURGEURL = f"https://api.bunny.net/pullzone/{PULLZONEID}/purgeCache"
- APICMD = f"curl -H 'AccessKey: {APIPASSWORD}' -s"
+ storage_conn_ = [
+ http.client.HTTPSConnection("sg.storage.bunnycdn.com") for _ in range(16)
+ ]
+ api_conn = http.client.HTTPSConnection("api.bunny.net")
+ conn_ready = asyncio.gather(
+ *(
+ asyncio.get_event_loop().run_in_executor(None, c.connect)
+ for c in storage_conn_
+ ),
+ asyncio.get_event_loop().run_in_executor(None, api_conn.connect),
+ )
+
+ storage_conn = asyncio.Queue()
+ for c in storage_conn_:
+ storage_conn.put_nowait(c)
@once()
def build_git_repo(
@@ -123,16 +135,43 @@ async def main():
async def contents(path: str):
async with bunny_sem:
print("+++ download", path)
- path_json = await shell(f"{STORAGECMD} '{STORAGEURL}/{path}/'")
+
+ c = await storage_conn.get()
+ c.request(
+ "GET",
+ f"/{STORAGENAME}/{path}/",
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ await asyncio.sleep(0)
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 200:
+ print("!!! download", resp.status, resp.reason, resp_body)
+ path_json = resp_body.decode("utf-8")
+ storage_conn.put_nowait(c)
+
print("--- download", path)
- return json.loads(path_json.utf8stdout)
+ return json.loads(path_json)
@once()
async def cleanfile(path: str):
if not os.path.isfile(f"{LOCALPATH}/{path}"):
async with bunny_sem:
print("+++ cleanfile", path)
- await shell(f"{STORAGECMD} -XDELETE '{STORAGEURL}/{path}'")
+
+ c = await storage_conn.get()
+ c.request(
+ "DELETE",
+ f"/{STORAGENAME}/{path}",
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ await asyncio.sleep(0)
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 200:
+ print("!!! cleanfile", resp.status, resp.reason, resp_body)
+ storage_conn.put_nowait(c)
+
print("--- cleanfile", path)
@once()
@@ -140,7 +179,20 @@ async def main():
if not os.path.isdir(f"{LOCALPATH}/{path}"):
async with bunny_sem:
print("+++ cleandir", path)
- await shell(f"{STORAGECMD} -XDELETE '{STORAGEURL}/{path}/'")
+
+ c = await storage_conn.get()
+ c.request(
+ "DELETE",
+ f"/{STORAGENAME}/{path}/",
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ await asyncio.sleep(0)
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 200:
+ print("!!! cleandir", resp.status, resp.reason, resp_body)
+ storage_conn.put_nowait(c)
+
print("--- cleandir", path)
@once()
@@ -181,9 +233,22 @@ async def main():
if bunny_checksum != our_checksum:
async with bunny_sem:
print("+++ uploading", path)
- await shell(
- f"{STORAGECMD} -T'{LOCALPATH}/{path}' '{STORAGEURL}/{path}'"
- )
+
+ c = await storage_conn.get()
+ with open(f"{LOCALPATH}/{path}", "rb") as f:
+ c.request(
+ "PUT",
+ f"/{STORAGENAME}/{path}",
+ body=f,
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ await asyncio.sleep(0)
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 201:
+ print("!!! uploading", resp.status, resp.reason, resp_body)
+ storage_conn.put_nowait(c)
+
print("--- uploading", path)
# print("- upload", path)
@@ -191,7 +256,17 @@ async def main():
async def purge():
async with bunny_sem:
print("+++ purge")
- await shell(f"{APICMD} -XPOST '{PURGEURL}'")
+
+ api_conn.request(
+ "POST",
+ f"/pullzone/{PULLZONEID}/purgeCache",
+ headers={"AccessKey": APIPASSWORD},
+ )
+ resp = api_conn.getresponse()
+ resp_body = resp.read()
+ if resp.status != 204:
+ print("!!! purge", resp.status, resp.reason, resp_body)
+
print("--- purge")
@once()
@@ -199,6 +274,7 @@ async def main():
await rebuild()
UPLOAD = (await shell(f"cd '{LOCALPATH}' && find . -type f")).utf8stdout
CLEAN = (await shell(f"cd '{LOCALPATH}' && find . -type d")).utf8stdout
+ await conn_ready
await asyncio.gather(
*((upload(path)) for path in UPLOAD.strip().split("\n")),
*((clean(path)) for path in CLEAN.strip().split("\n")),
diff --git a/make3/helpers.py b/make3/helpers.py
@@ -10,26 +10,36 @@ async def file_modtime(f: int | str | bytes | os.PathLike[str] | os.PathLike[byt
@once()
-@cache_conditionally(lambda f, *args: (f.name, *args))
-async def _file_hash(f: open, skip_if_modtime_matches=True):
- if skip_if_modtime_matches:
- rerun_if_changed()(await file_modtime(f.fileno()))
- else:
- rerun_always()
- h = hashlib.sha256()
- while True:
- chunk = f.read1()
- if chunk:
- h.update(chunk)
- else:
- break
- d = h.hexdigest()
- # print("hash", f.name, d)
- return d
-
-
+@cache_conditionally(lambda f, *args: (f.name if isinstance(f, open) else f, *args))
async def file_hash(f: open | bytes | str, skip_if_modtime_matches=True):
if isinstance(f, bytes) or isinstance(f, str):
with open(f, "rb") as _f:
- return await _file_hash(_f, skip_if_modtime_matches)
- return await _file_hash(f, skip_if_modtime_matches)
+ if skip_if_modtime_matches:
+ rerun_if_changed()(await file_modtime(_f.fileno()))
+ else:
+ rerun_always()
+ h = hashlib.sha256()
+ while True:
+ chunk = _f.read1()
+ if chunk:
+ h.update(chunk)
+ else:
+ break
+ d = h.hexdigest()
+ # print("hash", f.name, d)
+ return d
+ else:
+ if skip_if_modtime_matches:
+ rerun_if_changed()(await file_modtime(f.fileno()))
+ else:
+ rerun_always()
+ h = hashlib.sha256()
+ while True:
+ chunk = f.read1()
+ if chunk:
+ h.update(chunk)
+ else:
+ break
+ d = h.hexdigest()
+ # print("hash", f.name, d)
+ return d