commit 9d47e81fc5be8b7d28619c7e040e7aad46e58943
parent c23c0c74c132b7369c298b5270c8d383b94cc212
Author: gracefu <81774659+gracefuu@users.noreply.github.com>
Date: Thu, 24 Apr 2025 09:27:18 +0800
Use add_reader to revive coroutines
Diffstat:
| M | make.py | | | 177 | +++++++++++++++++++++++++++++++++++++++++-------------------------------------- |
1 file changed, 93 insertions(+), 84 deletions(-)
diff --git a/make.py b/make.py
@@ -1,8 +1,8 @@
from make3 import EchoAll, file_hash, make_main, once, shell
import asyncio
+import http.client
import json
import os
-import http.client
async def main():
@@ -16,7 +16,7 @@ async def main():
LOCALPATH = "public"
storage_conn_ = [
- http.client.HTTPSConnection("sg.storage.bunnycdn.com") for _ in range(16)
+ http.client.HTTPSConnection("sg.storage.bunnycdn.com") for _ in range(32)
]
api_conn = http.client.HTTPSConnection("api.bunny.net")
conn_ready = asyncio.gather(
@@ -85,12 +85,12 @@ async def main():
await shell(
"""
rm -rf public
+ mkdir -p public
"""
)
await asyncio.gather(
shell(
"""
- mkdir -p public
cp 404.html public
cp -a icons public
"""
@@ -129,71 +129,78 @@ async def main():
echo=EchoAll,
)
- bunny_sem = asyncio.Semaphore(80)
-
@once()
async def contents(path: str):
- async with bunny_sem:
- print("+++ download", path)
+ c: http.client.HTTPSConnection = await storage_conn.get()
+ print("+++ download", path)
+
+ c.request(
+ "GET",
+ f"/{STORAGENAME}/{path}/",
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ loop = asyncio.get_event_loop()
+ f = loop.create_future()
+ loop.add_reader(c.sock.fileno(), f.set_result, None)
+ await f
+ loop.remove_reader(c.sock.fileno())
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 200:
+ print("!!! download", resp.status, resp.reason, resp_body)
+ path_json = resp_body.decode("utf-8")
+
+ storage_conn.put_nowait(c)
+ print("--- download", path)
+ return json.loads(path_json)
+ @once()
+ async def cleanfile(path: str):
+ if not os.path.isfile(f"{LOCALPATH}/{path}"):
c = await storage_conn.get()
+ print("+++ cleanfile", path)
+
c.request(
- "GET",
- f"/{STORAGENAME}/{path}/",
+ "DELETE",
+ f"/{STORAGENAME}/{path}",
headers={"AccessKey": STORAGEPASSWORD},
)
- await asyncio.sleep(0)
+ loop = asyncio.get_event_loop()
+ f = loop.create_future()
+ loop.add_reader(c.sock.fileno(), f.set_result, None)
+ await f
+ loop.remove_reader(c.sock.fileno())
resp = c.getresponse()
resp_body = resp.read()
if resp.status != 200:
- print("!!! download", resp.status, resp.reason, resp_body)
- path_json = resp_body.decode("utf-8")
- storage_conn.put_nowait(c)
-
- print("--- download", path)
- return json.loads(path_json)
+ print("!!! cleanfile", resp.status, resp.reason, resp_body)
- @once()
- async def cleanfile(path: str):
- if not os.path.isfile(f"{LOCALPATH}/{path}"):
- async with bunny_sem:
- print("+++ cleanfile", path)
-
- c = await storage_conn.get()
- c.request(
- "DELETE",
- f"/{STORAGENAME}/{path}",
- headers={"AccessKey": STORAGEPASSWORD},
- )
- await asyncio.sleep(0)
- resp = c.getresponse()
- resp_body = resp.read()
- if resp.status != 200:
- print("!!! cleanfile", resp.status, resp.reason, resp_body)
- storage_conn.put_nowait(c)
-
- print("--- cleanfile", path)
+ storage_conn.put_nowait(c)
+ print("--- cleanfile", path)
@once()
async def cleandir(path: str):
if not os.path.isdir(f"{LOCALPATH}/{path}"):
- async with bunny_sem:
- print("+++ cleandir", path)
+ c = await storage_conn.get()
+ print("+++ cleandir", path)
- c = await storage_conn.get()
- c.request(
- "DELETE",
- f"/{STORAGENAME}/{path}/",
- headers={"AccessKey": STORAGEPASSWORD},
- )
- await asyncio.sleep(0)
- resp = c.getresponse()
- resp_body = resp.read()
- if resp.status != 200:
- print("!!! cleandir", resp.status, resp.reason, resp_body)
- storage_conn.put_nowait(c)
+ c.request(
+ "DELETE",
+ f"/{STORAGENAME}/{path}/",
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ loop = asyncio.get_event_loop()
+ f = loop.create_future()
+ loop.add_reader(c.sock.fileno(), f.set_result, None)
+ await f
+ loop.remove_reader(c.sock.fileno())
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 200:
+ print("!!! cleandir", resp.status, resp.reason, resp_body)
- print("--- cleandir", path)
+ storage_conn.put_nowait(c)
+ print("--- cleandir", path)
@once()
async def clean(path: str):
@@ -231,43 +238,45 @@ async def main():
our_checksum = (await file_hash(f"{LOCALPATH}/{path}")).upper()
if bunny_checksum != our_checksum:
- async with bunny_sem:
- print("+++ uploading", path)
-
- c = await storage_conn.get()
- with open(f"{LOCALPATH}/{path}", "rb") as f:
- c.request(
- "PUT",
- f"/{STORAGENAME}/{path}",
- body=f,
- headers={"AccessKey": STORAGEPASSWORD},
- )
- await asyncio.sleep(0)
- resp = c.getresponse()
- resp_body = resp.read()
- if resp.status != 201:
- print("!!! uploading", resp.status, resp.reason, resp_body)
- storage_conn.put_nowait(c)
-
- print("--- uploading", path)
+ c = await storage_conn.get()
+ print("+++ uploading", path)
+
+ with open(f"{LOCALPATH}/{path}", "rb") as f:
+ c.request(
+ "PUT",
+ f"/{STORAGENAME}/{path}",
+ body=f,
+ headers={"AccessKey": STORAGEPASSWORD},
+ )
+ loop = asyncio.get_event_loop()
+ f = loop.create_future()
+ loop.add_reader(c.sock.fileno(), f.set_result, None)
+ await f
+ loop.remove_reader(c.sock.fileno())
+ resp = c.getresponse()
+ resp_body = resp.read()
+ if resp.status != 201:
+ print("!!! uploading", resp.status, resp.reason, resp_body)
+
+ storage_conn.put_nowait(c)
+ print("--- uploading", path)
# print("- upload", path)
@once()
async def purge():
- async with bunny_sem:
- print("+++ purge")
+ print("+++ purge")
- api_conn.request(
- "POST",
- f"/pullzone/{PULLZONEID}/purgeCache",
- headers={"AccessKey": APIPASSWORD},
- )
- resp = api_conn.getresponse()
- resp_body = resp.read()
- if resp.status != 204:
- print("!!! purge", resp.status, resp.reason, resp_body)
+ api_conn.request(
+ "POST",
+ f"/pullzone/{PULLZONEID}/purgeCache",
+ headers={"AccessKey": APIPASSWORD},
+ )
+ resp = api_conn.getresponse()
+ resp_body = resp.read()
+ if resp.status != 204:
+ print("!!! purge", resp.status, resp.reason, resp_body)
- print("--- purge")
+ print("--- purge")
@once()
async def all():
@@ -276,8 +285,8 @@ async def main():
CLEAN = (await shell(f"cd '{LOCALPATH}' && find . -type d")).utf8stdout
await conn_ready
await asyncio.gather(
- *((upload(path)) for path in UPLOAD.strip().split("\n")),
- *((clean(path)) for path in CLEAN.strip().split("\n")),
+ *((upload(path)) for path in UPLOAD.strip().split("\n") if path),
+ *((clean(path)) for path in CLEAN.strip().split("\n") if path),
)
await purge()