diff --git a/docs/mri.md b/docs/mri.md index 502cf4ff3..cf2dd63c4 100644 --- a/docs/mri.md +++ b/docs/mri.md @@ -197,4 +197,8 @@ leaderboard = benchmark.leaderboards[0] # Get the standings standings = leaderboard.get_standings() # Returns a pandas dataframe + +# Access the jobs that ran for this leaderboard +for job in leaderboard.jobs: + results = job.get_results() ``` diff --git a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py index 9eec76948..922792383 100644 --- a/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py +++ b/src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: import pandas as pd + from rapidata.rapidata_client.job.rapidata_job import RapidataJob class RapidataLeaderboard: @@ -150,6 +151,51 @@ def name(self, name: str): self.__name = name self._update_config() + @property + def jobs(self) -> list[RapidataJob]: + """ + Returns all Rapidata jobs that have run for this leaderboard. + + Every model evaluation on the leaderboard is carried out by a job. This + collects the jobs across all runs of the leaderboard, most recent first. + + Returns: + A list of RapidataJob instances, one per run that has an associated job. + """ + with tracer.start_as_current_span("RapidataLeaderboard.jobs"): + from rapidata.rapidata_client.job.rapidata_job_manager import ( + RapidataJobManager, + ) + + job_manager = RapidataJobManager(self.__openapi_service) + + current_page = 1 + job_ids: list[str] = [] + + while True: + runs_result = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_runs_get( + leaderboard_id=self.id, + page=current_page, + page_size=100, + sort=["-created_at"], + ) + + if runs_result.total_pages is None: + raise ValueError( + "An error occurred while fetching runs: total_pages is None" + ) + + job_ids.extend( + run.job_id for run in runs_result.items if run.job_id is not None + ) + + if current_page >= runs_result.total_pages: + break + + current_page += 1 + + return [job_manager.get_job_by_id(job_id) for job_id in job_ids] + def get_standings(self, tags: Optional[list[str]] = None) -> "pd.DataFrame": """ Returns the standings of the leaderboard.