diff --git a/src/tokenization/claude-ipc.ts b/src/tokenization/claude-ipc.ts index 9601ffd..98975c6 100644 --- a/src/tokenization/claude-ipc.ts +++ b/src/tokenization/claude-ipc.ts @@ -112,7 +112,8 @@ function onMessage(requestId: Buffer, tokens: Buffer) { async function launchTokenizer() { return new Promise((resolve, reject) => { let resolved = false; - const proc = spawn("python", [ + const python = process.platform === "win32" ? "python" : "python3"; + const proc = spawn(python, [ "-u", join(__dirname, "tokenization", "claude-tokenizer.py"), ]); @@ -125,6 +126,13 @@ async function launchTokenizer() { proc.stderr!.on("data", (data) => { pythonLog.error(data.toString()); }); + proc.on("error", (err) => { + pythonLog.error({ err }, "Claude tokenizer error"); + if (!resolved) { + resolved = true; + reject(err); + } + }); proc.on("close", (code) => { pythonLog.info(`Claude tokenizer exited with code ${code}`); socket?.close(); diff --git a/src/tokenization/claude-tokenizer.py b/src/tokenization/claude-tokenizer.py index e09d472..06d7d66 100644 --- a/src/tokenization/claude-tokenizer.py +++ b/src/tokenization/claude-tokenizer.py @@ -19,8 +19,6 @@ def init(socket): while True: message = socket.recv_multipart() routing_id, command = message - print(f"claude-tokenizer.py: received message {message}") - print(f"claude-tokenizer.py: received command {command}") if command == b"init": print("claude-tokenizer.py: initialized") socket.send_multipart([routing_id, b"ok"])