Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ without any human review or supervision! The LLM could generate dangerous python
that harms your system, like this: `import os; os.system("sudo rm -rf /")`.
Running inside a docker container helps limit the damage that could be done.

The current harness (`benchmark.py`) organizes the exercises as "Cats": each
exercise carries `cat.yaml` metadata, enabling per-test selection and
deterministic subsets (e.g. `--hash-re '^0'`) for spot-checking models. The
canonical exercises source is
[ErichBSchulz/cecli-cats](https://github.com/ErichBSchulz/cecli-cats), a
Cats-formatted conversion of the polyglot benchmark.

## Usage

There are 3 main tasks involved in benchmarking:
Expand All @@ -41,17 +48,18 @@ There are 3 main tasks involved in benchmarking:
These steps only need to be done once.

```
ORG=Aider-AI
REPO=aider
ORG=cecli-dev
REPO=cecli
# Clone the main repo
git clone https://github.com/$ORG/$REPO.git

# Create the scratch dir to hold benchmarking results inside the main repo:
cd $REPO
mkdir tmp.benchmarks

# Clone the repo with the exercises
git clone https://github.com/$ORG/polyglot-benchmark tmp.benchmarks/polyglot-benchmark
# Clone the canonical exercises source for the Cats harness
git clone https://github.com/ErichBSchulz/cecli-cats tmp.benchmarks/cecli-cats


# Build the docker container
./benchmark/docker_build.sh
Expand All @@ -69,10 +77,10 @@ Launch the docker container and run the benchmark inside it:
./benchmark/docker.sh

# Run the benchmark:
./benchmark/benchmark.py a-helpful-name-for-this-run --model gpt-3.5-turbo --edit-format whole --threads 10 --exercises-dir polyglot-benchmark
./benchmark/benchmark.py a-helpful-name-for-this-run --model gpt-3.5-turbo --edit-format whole --threads 10 --exercises-dir cecli-cats

# Or with OpenRouter models (requires OPENROUTER_API_KEY environment variable):
./benchmark/benchmark.py openrouter-run --model openrouter/deepseek/deepseek-r1:free --edit-format whole --threads 10 --exercises-dir polyglot-benchmark
./benchmark/benchmark.py openrouter-run --model openrouter/deepseek/deepseek-r1:free --edit-format whole --threads 10 --exercises-dir cecli-cats
```

The above will create a folder
Expand Down Expand Up @@ -156,6 +164,7 @@ Note the roadmap priorities:

## Limitations

- `benchmark_classic.py` is deprecated (it predates the async Coder API).
- These scripts are not intended for use by typical `cecli` end users.
- Some of the old (?deprecated) tools are written as `bash` scripts, so it will be hard to use
them on Windows.
Expand Down
2 changes: 1 addition & 1 deletion cecli/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from packaging import version

__version__ = "0.100.11.dev"
__version__ = "0.100.14.dev"
safe_version = __version__

try:
Expand Down
6 changes: 6 additions & 0 deletions cecli/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,12 @@ def get_parser(default_config_files, git_root):
default=True,
help="Enable/disable streaming responses (default: True)",
)
group.add_argument(
"--spinner",
action=argparse.BooleanOptionalAction,
default=True,
help="Enable/disable the spinner while waiting for LLM responses (default: True)",
)
group.add_argument(
"--user-input-color",
default="#00cc00",
Expand Down
19 changes: 19 additions & 0 deletions cecli/coders/agent_coder.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,25 @@ async def _exec_async():
call_result = await litellm.experimental_mcp_client.call_openai_tool(
session=session, openai_tool=tool_call_dict
)
except Exception as e:
if server.is_session_expired_error(e):
try:
session = await server.reconnect()
call_result = await litellm.experimental_mcp_client.call_openai_tool(
session=session, openai_tool=tool_call_dict
)
except Exception as retry_exc:
self.io.tool_warning(
f"Executing {tool_name} on {server.name} failed after reconnect:\n"
f"Error: {retry_exc}"
)
return f"Error executing tool call {tool_name}: {retry_exc}"
else:
self.io.tool_warning(
f"Executing {tool_name} on {server.name} failed:\nError: {e}"
)
return f"Error executing tool call {tool_name}: {e}"
try:
content_parts = []
if call_result.content:
for item in call_result.content:
Expand Down
37 changes: 32 additions & 5 deletions cecli/coders/base_coder.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ def total_cached_tokens(self, value):
message_tokens_sent = 0
message_tokens_received = 0
message_cached_tokens = 0
message_cost_deferred = None
add_cache_headers = False
cache_warming_thread = None
num_cache_warming_pings = 0
Expand Down Expand Up @@ -1497,6 +1498,10 @@ async def _run_linear(self, with_message=None, preproc=True):
self.show_announcements()
self.suppress_announcements_for_next_prompt = True

if self.message_cost_deferred and not self.io.spinner_active:
self.io.tool_output(self.message_cost_deferred)
self.message_cost_deferred = None

await self.io.recreate_input()
await self.io.input_task
user_message = self.io.input_task.result()
Expand Down Expand Up @@ -1645,6 +1650,10 @@ async def input_task(self, preproc):
self.show_announcements()
self.suppress_announcements_for_next_prompt = True

if self.message_cost_deferred and not self.io.spinner_active:
self.io.tool_output(self.message_cost_deferred)
self.message_cost_deferred = None

# Stop spinner before showing announcements or getting input
self.io.stop_spinner()
self.copy_context()
Expand Down Expand Up @@ -2512,7 +2521,11 @@ async def format_in_executor():
if not self.tui:
spinner_text += f" • ${self.format_cost(self.total_cost)} session"

self.io.start_spinner(spinner_text, coder_uuid=getattr(self, "uuid", None))
if self.io.spinner_active:
self.io.start_spinner(spinner_text, coder_uuid=getattr(self, "uuid", None))
else:
self.message_cost_deferred = spinner_text

if self.stream:
self.mdstream = True
else:
Expand Down Expand Up @@ -2635,6 +2648,10 @@ async def format_in_executor():

# Ensure any waiting spinner is stopped
self.io.start_spinner("Processing Answer...", coder_uuid=getattr(self, "uuid", None))

if not self.io.spinner_active:
self.partial_response_content = self.get_multi_response_content_in_progress(True)

self.remove_reasoning_content()
self.multi_response_content = ""

Expand Down Expand Up @@ -2980,12 +2997,22 @@ async def _execute_mcp_tools(self, server, tool_calls):
continue

async def do_tool_call():
nonlocal session
from litellm import experimental_mcp_client

return await experimental_mcp_client.call_openai_tool(
session=session,
openai_tool=new_tool_call,
)
try:
return await experimental_mcp_client.call_openai_tool(
session=session,
openai_tool=new_tool_call,
)
except Exception as e:
if server.is_session_expired_error(e):
session = await server.reconnect()
return await experimental_mcp_client.call_openai_tool(
session=session,
openai_tool=new_tool_call,
)
raise

call_result, interrupted = await coroutines.interruptible(
do_tool_call(), self.interrupt_event
Expand Down
9 changes: 8 additions & 1 deletion cecli/interruptible_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,14 @@ def __init__(self):
raise RuntimeError("InterruptibleInput is Unix-only (requires selectable stdin).")

self._cancel = threading.Event()
self._sel = selectors.DefaultSelector()

# The default selector (Kqueue on macOS, Epoll on Linux) cannot
# handle pipe-based stdin (e.g. when running inside Emacs comint-mode).
# Fall back to SelectSelector which works with any fd that supports select().
if not sys.stdin.isatty():
self._sel = selectors.SelectSelector()
else:
self._sel = selectors.DefaultSelector()

# self-pipe to wake up select() from interrupt()
self._r, self._w = os.pipe()
Expand Down
15 changes: 14 additions & 1 deletion cecli/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ def __init__(
notifications_command=None,
notification_bell=False,
verbose=False,
show_spinner=True,
):
self.console = Console()
self.pretty = pretty
Expand Down Expand Up @@ -499,6 +500,7 @@ def __init__(
fancy_input = False

# Spinner state
self.spinner_active = show_spinner
self.spinner_running = False
self.spinner_text = ""
self.last_spinner_text = ""
Expand All @@ -507,7 +509,7 @@ def __init__(
self.spinner_last_frame_index = 0
self.unicode_palette = "░█"
self.fallback_spinner = None
self.fallback_spinner_enabled = True
self.fallback_spinner_enabled = show_spinner

self.interruptible_input = None

Expand Down Expand Up @@ -569,7 +571,12 @@ def start_spinner(self, text, update_last_text=True, **kwargs):
"""Start the spinner."""
self.stop_spinner()

if not self.spinner_active:
return

if self.prompt_session:
if not self.fallback_spinner_enabled:
return
self.spinner_running = True
self.spinner_text = text
self.spinner_frame_index = self.spinner_last_frame_index
Expand All @@ -582,9 +589,15 @@ def start_spinner(self, text, update_last_text=True, **kwargs):
self.fallback_spinner.step()

def update_spinner(self, text):
if not self.spinner_active:
return

self.spinner_text = text

def update_spinner_suffix(self, text=None):
if not self.spinner_active:
return

if text:
self.spinner_suffix = f" • {text[:16].strip()}"
else:
Expand Down
15 changes: 15 additions & 0 deletions cecli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@
if sys.platform == "win32":
if hasattr(asyncio, "set_event_loop_policy"):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
elif sys.platform == "darwin":
# The default KqueueSelector cannot handle pipe-based stdin
# (e.g. when running inside Emacs comint-mode). Fall back to
# SelectSelector which works with any file descriptor that supports select().
import selectors

if not sys.stdin.isatty():
_original_event_loop_policy = asyncio.DefaultEventLoopPolicy

class _SelectSelectorPolicy(asyncio.DefaultEventLoopPolicy):
def new_event_loop(self):
return asyncio.SelectorEventLoop(selectors.SelectSelector())

asyncio.set_event_loop_policy(_SelectSelectorPolicy())
from prompt_toolkit.enums import EditingMode

from .dump import dump # noqa
Expand Down Expand Up @@ -708,6 +722,7 @@ def get_io(pretty):
notifications_command=args.notifications_command,
notification_bell=args.notification_bell,
verbose=args.verbose,
show_spinner=args.spinner,
)

validate_tui_args(args)
Expand Down
60 changes: 44 additions & 16 deletions cecli/mcp/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,17 +160,46 @@ async def connect_server(self, name: str) -> bool:
self._server_tools[server.name] = get_local_tool_schemas()
return True

try:
session = await server.connect()
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
self._server_tools[server.name] = tools
self._connected_servers.add(server)
self._log_verbose(f"Connected to MCP server: {name}")
return True
except (Exception, asyncio.CancelledError) as e:
if server.name != "unnamed-server":
self._log_error(f"Failed to connect to MCP server {name}: {e}")
return False
# Retry with exponential backoff for transient connection failures.
# Note: This also fixes a latent bug where asyncio.CancelledError was
# silently caught and treated as a connection failure. CancelledError is
# now re-raised to properly propagate cancellation.
# When io is None (e.g., during from_servers before IO is assigned),
# _log_warning and _log_error silently return — retries still happen
# but with no user-visible feedback. This is intentional.
max_retries = 3 if server.name != "unnamed-server" else 1
delay = 1.0
backoff = 2.0
max_delay = 30.0

for attempt in range(1, max_retries + 1):
try:
session = await server.connect()
tools = await experimental_mcp_client.load_mcp_tools(
session=session, format="openai"
)
self._server_tools[server.name] = tools
self._connected_servers.add(server)
self._log_verbose(f"Connected to MCP server: {name}")
return True
except asyncio.CancelledError:
raise
except Exception as e:
if attempt < max_retries and server.name != "unnamed-server":
self._log_warning(
f"Connection attempt {attempt} failed for {name}, "
f"retrying in {delay}s... ({e})"
)

await asyncio.sleep(delay)
delay = min(delay * backoff, max_delay)
else:
if server.name != "unnamed-server":
self._log_error(
f"Failed to connect to MCP server {name} "
f"after {max_retries} attempts: {e}"
)
return False

async def disconnect_server(self, name: str) -> bool:
"""
Expand Down Expand Up @@ -281,11 +310,10 @@ async def add_server_with_retry(
success = await mcp_manager.add_server(server, connect=False)
return (server, success)

for _attempt in range(max_retries):
success = await mcp_manager.add_server(server, connect=True)
if success:
return (server, True)
return (server, False)
# connect_server now has built-in retry logic, so we only need
# a single call here — no separate retry loop needed.
success = await mcp_manager.add_server(server, connect=True)
return (server, success)

tasks = []
for server in servers:
Expand Down
Loading
Loading