AIStoreHub Package Vault: Paper Orchestra
Package Slug: paper-orchestra
Archetype: multi_agent
Upstream Repository: https://github.com/google-research/paper-orchestra
Workspace Files & Specifications (43 files)
- aistorehub.json (483 bytes)
{ "name": "Paper Orchestra", "slug": "paper-orchestra", "version": "1.0.0", "type": "multi_agent", "description": "PaperOrchestra is a multi-agent framework for automated AI research paper writing.", "owner": "community", "repository": "catalog", "github_repo_url": "https://github.com/google-research/paper-orchestra", "tags": [ "trending", "google-research", "python", "github", "open-source" ], "imported_at": "2026-11-21T23:56:33.606Z" } - autoraters/agent_review.py (10093 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import json import numpy as np from utils.gemini_utils import call_gemini_with_text_prompt # --- AGENTREVIEW SPECIFIC CONSTANTS (From role_descriptions.py) --- SCORE_CALCULATION = { 10: "This study is among the top 2% of all papers. It is one of the most thorough I have seen. It changed my thinking on this topic. I would fight for it to be accepted", 8: "This study is among the top 10% of all papers. It provides sufficient support for all of its claims/arguments. Some extra experiments are needed, but not essential. The method is highly original and generalizable to various fields. It deepens the understanding of some phenomenons or lowers the barriers to an existing research direction", 6: "This study provides sufficient support for its major claims/arguments, some minor points may need extra support or details. The method is moderately original and generalizable to various relevant fields. The work it describes is not particularly interesting and/or novel, so it will not be a big loss if people don’t see it in this conference", 5: "Some of the main claims/arguments are not sufficiently supported, there are major technical/methodological problems. The proposed method is somewhat original and generalizable to various relevant fields. I am leaning towards rejection, but I can be persuaded if my co-reviewers think otherwise", 3: "This paper makes marginal contributions", 1: "This study is not yet sufficiently thorough to warrant publication or is not relevant to the conference", } AGENT_REVIEW_RUBRICS = ( f"* 10: {SCORE_CALCULATION[10]};\n" f"* 8: {SCORE_CALCULATION[8]};\n" f"* 6: {SCORE_CALCULATION[6]};\n" f"* 5: {SCORE_CALCULATION[5]};\n" f"* 3: {SCORE_CALCULATION[3]};\n" f"* 1: {SCORE_CALCULATION[1]}. " ) # --- PROMPT GENERATORS --- def get_agentreview_system_prompt( is_knowledgeable=True, is_responsible=True, is_benign=True ): """ Constructs the system prompt based on AgentReview's role_descriptions.py. Defaults to the 'Best Case' reviewer (Knowledgeable, Responsible, Benign). """ bio = "You are a reviewer. You write peer review of academic papers by evaluating their technical quality, originality, and clarity.\n\n" if is_knowledgeable: bio += "Knowledgeability: You are knowledgeable, with a strong background and a PhD degree in the subject areas related to this paper. You possess the expertise necessary to scrutinize and provide insightful feedback to this paper.\n\n" else: bio += "Knowledgeability: You are not knowledgeable and do not have strong background in the subject areas related to this paper.\n\n" if is_responsible: bio += "Responsibility: As a responsible reviewer, you highly responsibly write paper reviews. You meticulously assess a research paper's technical accuracy, innovation, and relevance. You thoroughly read the paper, critically analyze the methodologies, and carefully consider the paper's contribution to the field.\n\n" else: bio += "Responsibility: As a lazy reviewer, your reviews tend to be superficial and hastily done. Your assessments might overlook critical details, lack depth in analysis, fail to recognize novel contributions, or offer generic feedback.\n\n" if is_benign: bio += "Intention: As a benign reviewer, your approach to reviewing is guided by a genuine intention to aid authors in enhancing their work. You provide detailed, constructive feedback, aimed at both validating robust research and guiding authors to refine and improve their work. You are also critical of technical flaws in the paper.\n\n" else: bio += "Intention: As a mean reviewer, your reviewing style is often harsh and overly critical, with a tendency towards negative bias. Your reviews may focus excessively on faults, sometimes overlooking the paper's merits.\n\n" bio += f"## Rubrics for Overall Rating\n{AGENT_REVIEW_RUBRICS}" return bio # We adapt the instruction form to request JSON but strictly follow AgentReview's content structure. AGENTREVIEW_INSTRUCTIONS = """ Respond in the following format: THOUGHT: <THOUGHT> REVIEW JSON: ```json <JSON> ``` In <THOUGHT>, first briefly discuss your intuitions and reasoning for the evaluation. Detail your high-level arguments, necessary choices and desired outcomes of the review. Do not make generic comments here, but be specific to your current paper. Treat this as the note-taking phase of your review. In <JSON>, provide the review in JSON format with the following fields in the order: - "Summary": A summary of the paper content and its contributions. - "Strengths": A list of strengths of the paper. - "Weaknesses": A list of weaknesses of the paper. - "Originality": A rating from 1 to 4 (low, medium, high, very high). - "Quality": A rating from 1 to 4 (low, medium, high, very high). - "Clarity": A rating from 1 to 4 (low, medium, high, very high). - "Significance": A rating from 1 to 4 (low, medium, high, very high). - "Questions": A set of clarifying questions to be answered by the paper authors. - "Limitations": A set of limitations and potential negative societal impacts of the work. - "Ethical Concerns": A boolean value indicating whether there are ethical concerns. - "Soundness": A rating from 1 to 4 (poor, fair, good, excellent). - "Presentation": A rating from 1 to 4 (poor, fair, good, excellent). - "Contribution": A rating from 1 to 4 (poor, fair, good, excellent). - "Overall": A rating from 1 to 10 (very strong reject to award quality). - "Confidence": A rating from 1 to 5 (low, medium, high, very high, absolute). - "Decision": A decision that has to be one of the following: Accept, Reject. For the "Decision" field, don't use Weak Accept, Borderline Accept, Borderline Reject, or Strong Reject. Instead, only use Accept or Reject. This JSON will be automatically parsed, so ensure the format is precise.""" def perform_review_agentreview( text, model_name, num_review_ensemble=3, temperature=0.75, reviewer_system_prompt=None, review_instruction_form=AGENTREVIEW_INSTRUCTIONS, ): print(f"Start paper reviewing using model: `{model_name}`") # Generate the "Best Case" AgentReview persona if reviewer_system_prompt is None: reviewer_system_prompt = get_agentreview_system_prompt( is_knowledgeable=True, is_responsible=True, is_benign=True ) base_prompt = review_instruction_form base_prompt += f""" Here is the paper you are asked to review: ``` {text} ```""" if num_review_ensemble > 1: parsed_reviews = [] for i in range(num_review_ensemble): print(f"Generating ensemble review {i + 1}/{num_review_ensemble}...") response = call_gemini_with_text_prompt( prompt=base_prompt, model_name=model_name, generation_configs={ "temperature": temperature, "system_instruction": reviewer_system_prompt, }, ) parsed_reviews.append(response["parsed_response"]) review = get_meta_review(model_name, temperature, parsed_reviews) if review is None: review = parsed_reviews[0] for score, limits in [ ("Originality", (1, 4)), ("Quality", (1, 4)), ("Clarity", (1, 4)), ("Significance", (1, 4)), ("Soundness", (1, 4)), ("Presentation", (1, 4)), ("Contribution", (1, 4)), ("Overall", (1, 10)), ("Confidence", (1, 5)), ]: scores = [] for r in parsed_reviews: if score in r: try: val = int(r[score]) if limits[0] <= val <= limits[1]: scores.append(val) except (ValueError, TypeError): pass if scores: review[score] = int(round(np.mean(scores))) else: response = call_gemini_with_text_prompt( prompt=base_prompt, model_name=model_name, generation_configs={ "temperature": temperature, "system_instruction": reviewer_system_prompt, }, ) review = response["parsed_response"] return review meta_reviewer_system_prompt = """You are a very knowledgeable and experienced area chair at a top-tier machine learning conference. You are in charge of meta-reviewing a paper that was reviewed by {reviewer_count} reviewers. Your job is to aggregate the reviews into a single meta-review in the same format. You are an inclusive area chair. You tend to hear from all reviewers' opinions and combine them with your own judgments to make the final decision.""" def get_meta_review(model, temperature, reviews): review_text = "" for i, r in enumerate(reviews): review_text += f""" Review {i + 1}/{len(reviews)}: ``` {json.dumps(r)} ``` """ base_prompt = AGENTREVIEW_INSTRUCTIONS + review_text response = call_gemini_with_text_prompt( prompt=base_prompt, model_name=model, generation_configs={ "temperature": temperature, "system_instruction": meta_reviewer_system_prompt.format( reviewer_count=len(reviews) ), }, ) meta_review = response["parsed_response"] return meta_review - autoraters/citation_f1.py (31494 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import os import requests import json import time import os.path as osp from typing import List, Dict, Any from joblib import Parallel, delayed # type: ignore from tqdm import tqdm import concurrent.futures from threading import Lock from utils.pdf_utils import get_paper_references_from_pdf, load_paper from utils.common_utils import create_log_folder from utils.content_parsing_utils import extract_paper_title_from_citation from utils.llm_backend_utils import call_llm_with_text_prompt from dotenv import load_dotenv # type: ignore dot_file = os.path.join(os.path.dirname(__file__), "../.env") load_dotenv(dot_file) DATASET_DIR = os.path.join(os.path.dirname(__file__), "../datasets") S2_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY") print(f"Loaded S2 API Key for this project: {S2_API_KEY}") PRIORITY_PROMPT = """You are an expert academic reviewer. Read the following paper text and analyze its references. Your goal is to categorize the provided references into two priorities: - P0 (Must-Have): Core citations strictly necessary for the paper. These MUST include: * Baselines directly compared against in experiments * Datasets the paper utilizes or evaluates on * Core methods the paper is directly building upon or modifying * Metrics or standard numbers heavily relied upon and cited from another paper - P1 (Good-To-Have): Supplemental citations. These include: * Standard background references covering broad history * General related work that is not directly competing or built-upon * Minor implementations or utility tools mentioned in passing PAPER TEXT: {paper_text} REFERENCES LIST: {references_str} Please return ONLY a JSON dictionary where the keys are the exact reference numbers (e.g., "1", "2") and the values are either "P0" or "P1". Example output: ```json {{ "1": "P0", "2": "P1", "3": "P0" }} ``` """ class ReferenceF1V1Evaluator: def __init__( self, api_key: str = S2_API_KEY, log_folder: str = None, model_name: str = "gemini-3.1-pro-preview", ): # Semantic Scholar API endpoint self.S2_SEARCH_URL = "https://api.semanticscholar.org/graph/v1/paper/search" self.headers = {"X-API-KEY": api_key} if api_key else {} self.model_name = model_name if not log_folder: log_folder = create_log_folder(prefix="citation_autorater") self.log_folder = log_folder def _clean_text_to_list(self, raw_text: str) -> tuple[List[str], List[str]]: """ Parses the raw string containing [1]...[2]... into a list of citation strings. """ pattern = r"\[\d+\]" segments = re.split(pattern, raw_text) cleaned_refs = [ seg.strip().replace("\n", " ").replace(" ", " ") for seg in segments if seg.strip() ] # Parallel extraction of titles from citation strings cleaned_titles = Parallel(n_jobs=10, backend="threading")( delayed(extract_paper_title_from_citation)(ref) for ref in tqdm(cleaned_refs, desc="Extracting Titles from Text") ) return cleaned_refs, cleaned_titles def _fetch_paper_id(self, paper_title: str) -> dict: """ Queries Semantic Scholar with the raw citation string to find a Unique ID (paperId). """ if not paper_title: return None params = {"query": paper_title, "limit": 1, "fields": "paperId,title"} try: response = requests.get( self.S2_SEARCH_URL, headers=self.headers, params=params, timeout=20 ) if response.status_code == 200: data = response.json() if data["total"] > 0 and data["data"]: return { "fetched_paper_id": data["data"][0]["paperId"], "fetched_paper_title": data["data"][0]["title"], } elif response.status_code == 429: print("Rate limit hit. Sleeping for 2 seconds...") time.sleep(2) return self._fetch_paper_id(paper_title) except Exception as e: print(f"Error fetching ID for ref: {paper_title}... -> {e}") return None def _get_ids_for_ref_list(self, ref_list: List[str], filler: str = "-1"): """ Batch processes a list of strings into a set of unique IDs. """ ids_set = set() ids_list = [] titles_list = [] print(f"Processing {len(ref_list)} references via S2 API...") for ref in tqdm(ref_list, desc="Resolving IDs"): fetched_info = self._fetch_paper_id(ref) pid = fetched_info["fetched_paper_id"] if fetched_info else None fetched_title = ( fetched_info["fetched_paper_title"] if fetched_info else None ) if pid: ids_set.add(pid) ids_list.append(pid) titles_list.append(fetched_title) else: ids_list.append(filler) titles_list.append(filler) time.sleep(0.5) return ids_set, ids_list, titles_list def _get_citation_priorities( self, paper_text: str, cleaned_refs: List[str] ) -> Dict[str, str]: """ Uses LLM to classify references into P0 or P1. """ references_str = "\n".join( [f"[{i+1}] {ref}" for i, ref in enumerate(cleaned_refs)] ) prompt = PRIORITY_PROMPT.format( paper_text=paper_text, references_str=references_str ) try: output = call_llm_with_text_prompt( prompt=prompt, model_name=self.model_name, generation_configs={"temperature": 0.0}, ) parsed_priorities = output.get("parsed_response", {}) if not parsed_priorities: print("Warning: LLM returned empty or invalid priority JSON.") return parsed_priorities except Exception as e: print(f"Error calling LLM for priorities: {e}") return {} def process_text_to_data( self, raw_text: str, tag: str = "na", paper_text: str = None ) -> Dict[str, Any]: """ Orchestrates cleaning text and fetching IDs. If paper_text is provided, uses LLM to assign P0/P1 priorities. Returns a serializable dictionary. """ ref_list, ref_titles = self._clean_text_to_list(raw_text) filler_val = f"{tag}_NA" ids_set, ids_list, fetched_titles = self._get_ids_for_ref_list( ref_titles, filler=filler_val ) priorities = {} if paper_text is not None: print("Calling LLM to classify citation priorities...") # Map 1-indexed strictly to the extracted ref_list length start_llm = time.time() priorities = self._get_citation_priorities(paper_text, ref_list) end_llm = time.time() print(f"Finished Calling LLM. Took {end_llm - start_llm:.2f} seconds.") # Structure the citation info for logging/debugging citation_info = [] p0_ids = set() p1_ids = set() for idx in range(len(ref_list)): priority = ( priorities.get(str(idx + 1), "P1") if priorities else "P1" ) # Default to P1 if no priority provided or missing pid = ids_list[idx] if pid and pid != filler_val: if priority == "P0": p0_ids.add(pid) if pid in p1_ids: p1_ids.remove(pid) # P0 takes precedence else: if pid not in p0_ids: p1_ids.add(pid) citation_info.append( { "citation_text": ref_list[idx], "extracted_paper_title": ref_titles[idx], "fetched_paper_id": pid, "fetched_paper_title": fetched_titles[idx], "priority": priority, } ) return { "citation_info": citation_info, "all_ids": list(ids_set), "all_ids_count": len(ids_set), "p0_ids": list(p0_ids), "p0_ids_count": len(p0_ids), "p1_ids": list(p1_ids), "p1_ids_count": len(p1_ids), } def evaluate_from_data(self, gt_data: Dict, gen_data: Dict) -> Dict: """ Computes precision/recall/f1 using pre-processed dictionary data for Overall, P0, and P1. """ log_path_gt = os.path.join(self.log_folder, "gt_ref_info.json") with open(log_path_gt, "w") as f: json.dump(gt_data, f, indent=4) log_path_gen = os.path.join(self.log_folder, "gen_ref_info.json") with open(log_path_gen, "w") as f: json.dump(gen_data, f, indent=4) gt_ids_all = set(gt_data.get("all_ids", [])) gt_ids_p0 = set(gt_data.get("p0_ids", [])) gt_ids_p1 = set(gt_data.get("p1_ids", [])) # Fallback extraction if older cache doesn't have p0_ids list directly if not gt_ids_p0 and not gt_ids_p1 and "citation_info" in gt_data: gt_ids_all = set() for item in gt_data.get("citation_info", []): pid = item["fetched_paper_id"] if pid and pid != "gt_NA": gt_ids_all.add(pid) if item.get("priority") == "P0": gt_ids_p0.add(pid) if pid in gt_ids_p1: gt_ids_p1.remove(pid) else: if pid not in gt_ids_p0: gt_ids_p1.add(pid) # Still fallback to pairwise if cache is old elif not gt_ids_p0 and not gt_ids_p1 and "pairwise" in gt_data: gt_ids_all = set() for item in gt_data.get("pairwise", []): pid = item["fetched_paper_id"] if pid and pid != "gt_NA": gt_ids_all.add(pid) if item.get("priority") == "P0": gt_ids_p0.add(pid) if pid in gt_ids_p1: gt_ids_p1.remove(pid) else: if pid not in gt_ids_p0: gt_ids_p1.add(pid) gen_ids_all = set(gen_data.get("all_ids", [])) gen_ids_all = {pid for pid in gen_ids_all if pid and pid != "gen_NA"} print("-" * 50) print(f"Ground Truth Unique IDs (All): {len(gt_ids_all)}") print(f"Ground Truth Unique IDs (P0): {len(gt_ids_p0)}") print(f"Ground Truth Unique IDs (P1): {len(gt_ids_p1)}") print(f"Generated Unique IDs: {len(gen_ids_all)}") tp_overall = len(gen_ids_all.intersection(gt_ids_all)) gen_count = len(gen_ids_all) gt_count = len(gt_ids_all) gt_p0_count = len(gt_ids_p0) gt_p1_count = len(gt_ids_p1) tp_p0 = len(gen_ids_all.intersection(gt_ids_p0)) tp_p1 = len(gen_ids_all.intersection(gt_ids_p1)) overall_precision = tp_overall / gen_count if gen_count > 0 else 0.0 overall_recall = tp_overall / gt_count if gt_count > 0 else 0.0 overall_f1 = ( 2 * (overall_precision * overall_recall) / (overall_precision + overall_recall) if (overall_precision + overall_recall) > 0 else 0.0 ) p0_recall = tp_p0 / gt_p0_count if gt_p0_count > 0 else 0.0 p1_recall = tp_p1 / gt_p1_count if gt_p1_count > 0 else 0.0 metrics_dict = { "overall_precision": overall_precision, "overall_recall": overall_recall, "overall_f1": overall_f1, "p0_recall": p0_recall, "p1_recall": p1_recall, "counts": { "ground_truth_ids_all": gt_count, "ground_truth_ids_p0": gt_p0_count, "ground_truth_ids_p1": gt_p1_count, "generated_ids_all": gen_count, "matches_overall": tp_overall, "matches_p0": tp_p0, "matches_p1": tp_p1, }, } output_json_path = os.path.join(self.log_folder, "metrics.json") with open(output_json_path, "w") as f: json.dump(metrics_dict, f, indent=4) print(f"Dumped metrics to {output_json_path}.") return metrics_dict def get_citation_f1_metrics( original_pdf: str, generated_pdf: str, log_folder=None, gt_cache_path: str = None, model_name: str = "gemini-3.1-pro-preview", ): """ Computes priority-based citation F1. Caches Ground Truth processing to avoid redundant LLM/API calls. """ evaluator = ReferenceF1V1Evaluator(log_folder=log_folder, model_name=model_name) # --- 1. Handle Ground Truth (With Caching) --- if not gt_cache_path: gt_cache_path = ( os.path.splitext(original_pdf)[0] + f"_gt_citations_{model_name}.json" ) print(f"Using GT cache path: {gt_cache_path}") gt_data = None if os.path.exists(gt_cache_path): print(f"Loading cached Ground Truth references from: {gt_cache_path}") try: with open(gt_cache_path, "r") as f: gt_data = json.load(f) except Exception as e: print(f"Failed to load cache {gt_cache_path}, will re-process. Error: {e}") if gt_data is None: print(f"Processing Ground Truth PDF: {original_pdf}") gt_references_text = get_paper_references_from_pdf( original_pdf, model_name=model_name ) if gt_references_text is None: print("Error: Could not extract text from GT PDF.") gt_references_text = "" print(f"Loading full paper text for LLM citation categorization...") paper_text = load_paper( original_pdf, num_pages=20 ) # Limit to 20 pages to save token limit context if needed, though pro can handle more gt_data = evaluator.process_text_to_data( gt_references_text, tag="gt", paper_text=paper_text ) try: with - autoraters/lit_review_quality.py (11360 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import tqdm import threading from concurrent.futures import ThreadPoolExecutor, as_completed from pydantic import BaseModel, Field from pydantic import BaseModel, Field from utils.llm_backend_utils import call_llm_with_pdf from autoraters.prompts.lit_review_quality_prompts import ( lit_review_quality_system_prompt, ) # --- STRUCTURED OUTPUT SCHEMA --- class SectionReview(BaseModel): score: int = Field(..., ge=0, le=100, description="Score 0-100 based on the rubric") justification: str = Field( ..., description="Critical analysis citing specific text." ) class PaperReview(BaseModel): paper_title: str = Field(..., description="Extracted title of the paper") clarity: SectionReview technical_depth: SectionReview connectivity: SectionReview presentation: SectionReview overall_quality_score: int = Field(..., description="Holistic score (0-100)") summary: str = Field( ..., description="A brief summary of the overall quality score justification." ) # --- END STRUCTURED OUTPUT SCHEMA --- # Global lock for file writing FILE_LOCK = threading.Lock() def rate_single_paper( paper_pdf_path: str, model_name: str = "gemini-3.1-pro-preview", avg_citation_count: int = 58, ): try: with open(paper_pdf_path, "rb") as f: pdf_bytes = f.read() except FileNotFoundError: print(f"Error: File {paper_pdf_path} not found.") return "" max_retries = 5 cur_try = 0 succeed = False while (not succeed) and cur_try < max_retries: try: sys_instruct = lit_review_quality_system_prompt.format( avg_citation_count=avg_citation_count ) response_dict = call_llm_with_pdf( pdf_path=paper_pdf_path, prompt="Rate this paper.", model_name=model_name, system_instruction=sys_instruct, temperature=0.0, ) succeed = True return response_dict["parsed_response"] except Exception as e: print( f"Error processing {paper_pdf_path}: {e}. Retrying [{cur_try + 1}/{max_retries}]..." ) cur_try += 1 return "" def _process_and_save_single_paper( paper_id: str, paper_path: str, output_json_path: str, current_ratings_dict: dict, avg_citation_count: int, model_name: str, ): """ Helper function to run inside the thread. It rates the paper and immediately saves it to disk using a lock. """ lit_rating = rate_single_paper( paper_path, avg_citation_count=avg_citation_count, model_name=model_name ) if not lit_rating: return paper_id, False lit_rating["pdf_path"] = paper_path with FILE_LOCK: current_ratings_dict[paper_id] = lit_rating with open(output_json_path, "w") as f: json.dump(current_ratings_dict, f, indent=4) return paper_id, True def perform_batch_rating( paper_path_dict: dict[str, str], output_json_path: str, max_workers: int = 10, avg_citation_count: int = 58, model_name: str = "gemini-3.1-pro-preview", ): if os.path.exists(output_json_path): with open(output_json_path, "r") as f: try: all_lit_ratings = json.load(f) except json.JSONDecodeError: all_lit_ratings = {} print( f"Warning: {output_json_path} was corrupted or empty. Starting fresh." ) print( f"Loaded {len(all_lit_ratings)-1} existing literature review ratings from '{output_json_path}'." ) else: all_lit_ratings = {} papers_to_process = {} for paper_id, paper_path in paper_path_dict.items(): if paper_id in all_lit_ratings: continue papers_to_process[paper_id] = paper_path total_to_process = len(papers_to_process) if total_to_process == 0: print("All papers have already been rated. Nothing to do.") return all_lit_ratings print( f"Starting parallel rating for {total_to_process} papers with {max_workers} threads..." ) with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_paper = { executor.submit( _process_and_save_single_paper, pid, path, output_json_path, all_lit_ratings, avg_citation_count, model_name, ): pid for pid, path in papers_to_process.items() } for future in tqdm.tqdm( as_completed(future_to_paper), total=total_to_process, desc="Rating Papers" ): paper_id = future_to_paper[future] try: pid, success = future.result() if not success: print(f"Failed to rate paper: {pid}") except Exception as exc: print(f"Paper {paper_id} generated an exception: {exc}") print( f"All {len(all_lit_ratings)} literature review ratings saved to {output_json_path}." ) return all_lit_ratings def perform_batch_rating_folder( root_folder: str, paper_rel_path="latex_writeup/final_paper.pdf", avg_citation_count: int = 58, model_name: str = "gemini-3.1-pro-preview", ): assert os.path.exists(root_folder), "Please provide a valid folder!" all_lit_ratings_save_path = os.path.join( root_folder, f"lit_review_quality_ratings_{model_name}_{paper_rel_path.split('/')[-1].replace('.pdf', '')}.json", ) all_filename = os.listdir(root_folder) all_folders = [] for filename in all_filename: if os.path.isdir(os.path.join(root_folder, filename)): all_folders.append(filename) all_folders.sort() paper_path_dict = {} for folder_name in tqdm.tqdm(all_folders, desc="Scanning folders"): paper_id = folder_name.strip() paper_path = os.path.join(root_folder, folder_name, paper_rel_path) if os.path.exists(paper_path): paper_path_dict[paper_id] = paper_path print(f"Extracted {len(paper_path_dict.keys())} papers from {root_folder}.") all_lit_ratings = perform_batch_rating( paper_path_dict=paper_path_dict, output_json_path=all_lit_ratings_save_path, max_workers=10, # Adjustable based on API rate limits avg_citation_count=avg_citation_count, model_name=model_name, ) print_literature_review_stats(all_lit_ratings_save_path) return all_lit_ratings def print_literature_review_stats(file_path): """ Parses a JSON file containing literature review evaluations and prints statistical averages for axis scores and the overall score. Saves overall metrics back to the JSON. """ if not os.path.exists(file_path): print(f"Error: The file '{file_path}' was not found.") return try: with open(file_path, "r", encoding="utf-8") as f: data = json.load(f) except json.JSONDecodeError: print( f"Error: Failed to decode JSON from '{file_path}'. Please check the file format." ) return except Exception as e: print(f"An unexpected error occurred: {e}") return total_papers = 0 total_overall_score_sum = 0 axis_data = {} for paper_id, paper_content in data.items(): if paper_id == "overall_metrics": continue total_papers += 1 total_overall_score_sum += paper_content.get( "overall_score", paper_content.get("overall_quality_score", 0) ) axis_scores_dict = paper_content.get("axis_scores", {}) if not axis_scores_dict: for key in ["clarity", "technical_depth", "connectivity", "presentation"]: if ( key in paper_content and isinstance(paper_content[key], dict) and "score" in paper_content[key] ): if key not in axis_data: axis_data[key] = [] axis_data[key].append(paper_content[key]["score"]) else: for axis_name, axis_details in axis_scores_dict.items(): score = axis_details.get("score") if score is not None: if axis_name not in axis_data: axis_data[axis_name] = [] axis_data[axis_name].append(score) if total_papers == 0: print("No papers found in the JSON file.") return print("=" * 60) print(f"LITERATURE REVIEW EVALUATION STATS") print(f"Total Papers Processed: {total_papers}") print("=" * 60) print(f"{'Metric / Axis':<45} | {'Avg Score'}") print("-" * 60) overall_metrics = {} avg_overall = total_overall_score_sum / total_papers print(f"{'Overall Score':<45} | {avg_overall:.2f}") overall_metrics["overall_score"] = avg_overall print("-" * 60) for axis_name in sorted(axis_data.keys()): scores_list = axis_data[axis_name] avg_axis_score = sum(scores_list) / len(scores_list) formatted_name = axis_name.replace("_", " ").title() print(f"{formatted_name:<45} | {avg_axis_score:.2f}") overall_metrics[axis_name] = avg_axis_score print("=" * 60) data["overall_metrics"] = overall_metrics with open(file_path, "w") as f: json.dump(data, f, indent=4) def perform_batch_rating_json( paper_info_json_path: str, avg_citation_count: int = 58, model_name: str = "gemini-3.1-pro-preview", ): with open(paper_info_json_path, "r") as f: paper_infos = json.load(f) print(f"Loaded {len(paper_infos)} papers from {paper_info_json_path}") paper_path_dict = {} for paper_info in paper_infos: if not os.path.exists(paper_info["pdf_path"]): print(f"{paper_info['pdf_path']} does not exist, skipping...") continue else: paper_path_dict[paper_info["paper_id"]] = paper_info["pdf_path"] print( f"Extracted {len(paper_path_dict.keys())} papers from {paper_info_json_path}." ) output_json_path = paper_info_json_path.replace( ".json", "_lit_review_quality_ratings.json" ) all_lit_ratings = perform_batch_rating( paper_path_dict=paper_path_dict, output_json_path=output_json_path, max_workers=10, avg_citation_count=avg_citation_count, model_name=model_name, ) print_literature_review_stats(output_json_path) return all_lit_ratings - autoraters/prompts/lit_review_quality_prompts.py (7688 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. lit_review_quality_system_prompt = """You are an expert, skeptical academic reviewer agent. Your task is to rigorously evaluate the quality of the literature review in a draft research paper PDF. You must be conservative with scoring. High scores are rare and must be explicitly justified with concrete evidence from the text. Assume most drafts are not publication-ready. CONTEXTUAL BASELINE The user has provided the average citation count for accepted papers in this specific field/venue. Reference Average Citation Count: {avg_citation_count} Use this number as the baseline for "typical" coverage volume. SCOPE - Evaluate ONLY the literature-review function of: - Introduction - Related Work / Background / Literature Review (or equivalent) - Ignore methods, experiments, and results except to verify whether the literature review correctly sets up the paper’s scope and claims. PROCESS (FOLLOW STRICTLY) 1. Identify the paper title. 2. Locate the Introduction and Related Work sections (or closest equivalents). 3. Identify: - The paper’s stated research problem - Claimed contributions - Implied relevant subfields 4. Estimate citation statistics from the literature review: - Approximate number of unique cited works - Citation density relative to section length - Breadth across relevant sub-areas - Volume relative to the Reference Average ({avg_citation_count}). 5. For each scoring axis, evaluate ONLY what is explicitly written. - Do NOT infer author intent. - Do NOT reward missing but “expected” knowledge. 6. Apply anti-inflation rules and penalties. 7. Produce output strictly in the JSON schema defined below. - NO extra text before or after the JSON. - All fields must be filled. - Use null if information is genuinely unavailable. ANTI-INFLATION RULES (MANDATORY) - Default expectation: overall score between 45–70. - Scores >85 require strong evidence across ALL axes. - Scores >90 are extremely rare and require near-survey-level mastery. - If any axis <50, overall score should rarely exceed 75. - If the review is mostly descriptive (paper-by-paper summaries), Critical Analysis must be ≤60. - If novelty is asserted without explicit comparison to close prior work, Positioning must be ≤60. - Sparse or inconsistent citations cap Citation Rigor at ≤60. - High citation count does NOT automatically imply high quality; relevance and synthesis must justify it. SCORING SCALE (ANCHORS — DO NOT INVENT NEW ONES) 0–20 = Unacceptable 21–40 = Weak 41–55 = Adequate but flawed 56–70 = Solid 71–85 = Strong 86–92 = Excellent 93–100 = Exceptional (extremely rare) AXES (0–100 EACH) Axis 1: Coverage & Completeness Evaluate: - Breadth across major relevant threads - Inclusion of foundational and recent work - Absence of obvious omissions - Citation volume relative to the Reference Average ({avg_citation_count}) Citation count anchors (Relative to Reference Average of {avg_citation_count}): - Count is < 50% of Reference: Usually narrow or incomplete (cap ≤55 unless field is very small). - Count is 50%–80% of Reference: Minimal acceptable coverage. - Count is 80%–120% of Reference: Solid breadth if well integrated. - Count is > 120% of Reference: Strong evidence of comprehensive coverage IF relevance is maintained. Axis 2: Relevance & Focus Evaluate: - Alignment of citations with the research problem - Minimal tangents or citation padding - Clear scoping and prioritization of literature Axis 3: Critical Analysis & Synthesis Evaluate: - Thematic grouping and comparison of approaches - Discussion of tradeoffs, limitations, and open gaps - Evidence of synthesis rather than sequential summaries Hard cap: ≤60 if the review is mostly descriptive. Axis 4: Positioning & Novelty Justification Evaluate: - Clear, literature-grounded research gap - Explicit differentiation from closest related work - Motivation for why the gap matters Hard cap: ≤60 if novelty claims are vague or unsupported. Axis 5: Organization & Writing Quality Evaluate: - Logical structure, flow, and signposting - Clarity and precision of academic language - Appropriate subsectioning and definitions Axis 6: Citation Practices, Density & Scholarly Rigor Evaluate: - Whether key claims are supported by citations - Credibility and consistency of sources - Citation density relative to section length - Balance between foundational and recent work Hard caps: - Citation count significantly below Reference Average ({avg_citation_count}) for a broad problem: ≤55 - High citation count with weak integration: ≤65 PENALTIES (APPLY AFTER AXIS SCORING) Apply zero or more penalties: - Overclaiming novelty without close comparison: −5 to −15 - Missing key recent work (if detectable): −5 to −15 - Mostly descriptive review with weak synthesis: −5 to −10 - Weak or generic gap statements: −5 to −10 - Citation dumping or consistency issues: −5 to −10 OPTIONAL POSITIVE ADJUSTMENT (RARE) You MAY apply a small positive adjustment (+3 to +7 total points) ONLY IF: - Citation count is substantially higher (>150%) than the Reference Average ({avg_citation_count}) - Citations are relevant and distributed across subtopics - Review remains synthesized and focused - Critical Analysis score >60 AND Relevance score >65 Do NOT apply this adjustment otherwise. OVERALL SCORE - Use weighted judgment: - Coverage: 20% - Relevance: 15% - Critical Analysis: 25% - Positioning: 25% - Organization: 10% - Citation Rigor: 5% - Then apply penalties and any justified positive adjustment. - Sanity-check against anti-inflation rules. OUTPUT FORMAT (STRICT JSON ONLY) Return exactly the following JSON structure and nothing else: ```json {{ "paper_title": string | null, "citation_statistics": {{ "estimated_unique_citations": number, "citation_density_assessment": "low" | "appropriate" | "high", "breadth_across_subareas": "narrow" | "moderate" | "broad", "comparison_to_baseline": string, "notes": string }}, "axis_scores": {{ "coverage_and_completeness": {{ "score": number, "justification": string }}, "relevance_and_focus": {{ "score": number, "justification": string }}, "critical_analysis_and_synthesis": {{ "score": number, "justification": string }}, "positioning_and_novelty": {{ "score": number, "justification": string }}, "organization_and_writing": {{ "score": number, "justification": string }}, "citation_practices_and_rigor": {{ "score": number, "justification": string }} }}, "penalties": [ {{ "reason": string, "points_deducted": number }} ], "summary": {{ "strengths": [string], "weaknesses": [string], "top_improvements": [string] }}, "overall_score": number }} ``` JUSTIFICATION CONSTRAINTS - Each justification: 2–5 sentences, evidence-based. - Do NOT quote more than 25 total words from the paper. - If evidence is missing, explicitly state: “Not evidenced in the text.” """ - autoraters/prompts/sxs_quality_prompts.py (4552 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. sxs_lit_review_quality_system_prompt = """You are an expert AI researcher and reviewer for top-tier machine learning conferences (e.g., CVPR, NeurIPS, ICLR). Your task is to perform a Side-by-Side (SxS) comparison of the literature review sections (Introduction and Related Work) between two academic papers. The ordering of the papers is arbitrary and does not indicate quality. Evaluate each paper independently before comparing them. Do not base your decision solely on length or verbosity. CRITICAL EVALUATION CRITERIA: 1. Problem Framing and Motivation - Which paper introduces the research problem more clearly? - Does the introduction explain the importance of the problem and the gap in existing work? 2. Coverage of Prior Work - Which paper provides a more complete and relevant overview of prior research? 3. Organization and Synthesis - Which paper organizes related work more effectively (e.g., grouping by themes or approaches)? - Does it synthesize prior work rather than simply listing papers? 4. Positioning of the Contribution - Which paper more clearly explains how its approach differs from existing methods? 5. Writing Quality and Readability - Which literature review is clearer, more concise, and easier to follow? OUTPUT FORMAT: Return a valid JSON object with the following schema: ```json { "paper_1_analysis": "analysis of paper 1", "paper_2_analysis": "analysis of paper 2", "comparison_justification": "comparison reasoning", "winner": "winner of your choice" } ``` The "winner" field must be exactly one of: "paper_1", "paper_2", or "tie". """ sxs_paper_quality_system_prompt = """You are an expert AI researcher and reviewer for top-tier machine learning conferences (e.g., CVPR, NeurIPS, ICLR). Your task is to perform a Side-by-Side (SxS) holistic comparison of two academic papers. The two papers describe the same or highly similar research ideas. Your evaluation should formulate a holistic judgment that accounts for both scientific execution and writing quality/presentation. The ordering of the papers is arbitrary and does not indicate quality. Evaluate each paper independently before comparing them. Do not base your decision solely on length or verbosity. CRITICAL EVALUATION CRITERIA: 1. Scientific Depth and Soundness - Which paper provides more rigorous technical justifications, theoretical foundations, and comprehensive experimental setups? 2. Technical Execution - Within the bounds of the described idea, which paper executes the implementation and methodology more innovatively or effectively? 3. Organization and Logical Flow - Which paper presents ideas in a clearer and more coherent order from Abstract through Conclusion? - Are sections and paragraphs structured logically with smooth transitions? 4. Clarity and Precision of Writing - Which paper explains its ideas more clearly and concisely? - Does the writing avoid unnecessary verbosity, ambiguity, or repetitive phrasing? 5. Presentation of Evidence and Formatting - Which paper integrates figures, tables, and experimental results more effectively into the narrative? - Are visuals clearly referenced and explained in the text? - Which paper has fewer visual formatting mistakes (e.g., overflowed tables, misplaced figures, overlapping text)? 6. Professional Academic Style - Which paper maintains a more polished and professional academic tone? - Does it use precise domain terminology and consistent terminology throughout the paper? OUTPUT FORMAT: Return a valid JSON object with the following schema: ```json { "paper_1_holistic_analysis": "analysis of paper_1 writing, presentation, and scientific execution", "paper_2_holistic_analysis": "analysis of paper_2 writing, presentation, and scientific execution", "comparison_justification": "comparison reasoning", "winner": "winner of your choice" } ``` The "winner" field must be exactly one of: "paper_1", "paper_2", or "tie". """ - autoraters/sxs_lit_review_quality.py (11512 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import threading from concurrent.futures import ThreadPoolExecutor, as_completed from utils.pdf_utils import load_paper from utils.llm_backend_utils import call_llm_with_text_prompt from autoraters.prompts.sxs_quality_prompts import sxs_lit_review_quality_system_prompt # Global lock for file writing FILE_LOCK = threading.Lock() def rate_sxs_lit_review( paper1_text: str, paper2_text: str, model_name: str = "gemini-3.1-pro-preview" ): max_retries = 5 cur_try = 0 succeed = False prompt = f"Paper 1:\n{paper1_text}\n\n---\n\nPaper 2:\n{paper2_text}\n\nTask: Compare the literature review quality of these two papers side by side." while (not succeed) and cur_try < max_retries: try: if "gemini" in model_name: response_dict = call_llm_with_text_prompt( prompt=prompt, model_name=model_name, generation_configs={ "system_instruction": sxs_lit_review_quality_system_prompt, "temperature": 0.0, }, ) succeed = True return ( response_dict.get("parsed_response", response_dict) if isinstance(response_dict, dict) else response_dict ) elif "gpt" in model_name: response_dict = call_llm_with_text_prompt( prompt=prompt, model_name=model_name, generation_configs={ "system_instruction": sxs_lit_review_quality_system_prompt, }, ) succeed = True return ( response_dict.get("parsed_response", response_dict) if isinstance(response_dict, dict) else response_dict ) except Exception as e: print( f"Error processing SxS: {e}. Retrying [{cur_try + 1}/{max_retries}]...", flush=True, ) cur_try += 1 return {} def _process_and_save_single_sxs( paper_id: str, paper1_path: str, paper2_path: str, output_json_path: str, current_results_dict: dict, model_name: str, exp1_name: str, exp2_name: str, ): """ Helper function to run inside the thread. Rates SxS lit review twice (both orders) and aggregates. """ try: paper1_text = load_paper(paper1_path, min_size=100) except Exception as e: print(f"Error reading paper 1 {paper1_path}: {e}") return paper_id, False try: paper2_text = load_paper(paper2_path, min_size=100) except Exception as e: print(f"Error reading paper 2 {paper2_path}: {e}") return paper_id, False rating1 = rate_sxs_lit_review(paper1_text, paper2_text, model_name=model_name) rating2 = rate_sxs_lit_review(paper2_text, paper1_text, model_name=model_name) if not rating1 or not rating2: print( f"❌ [{paper_id}] Failed to get ratings for one or both rounds.", flush=True ) return paper_id, False winner1 = rating1.get("winner", "").lower() if isinstance(rating1, dict) else "" winner2 = rating2.get("winner", "").lower() if isinstance(rating2, dict) else "" r1_result = "tie" if "paper_1" in winner1 or "paper 1" in winner1: r1_result = "p1" elif "paper_2" in winner1 or "paper 2" in winner1: r1_result = "p2" r2_result = "tie" if "paper_1" in winner2 or "paper 1" in winner2: r2_result = "p2" # Swap! elif "paper_2" in winner2 or "paper 2" in winner2: r2_result = "p1" s1 = 1 if r1_result == "p1" else (-1 if r1_result == "p2" else 0) s2 = 1 if r2_result == "p1" else (-1 if r2_result == "p2" else 0) total_score = s1 + s2 category_5pt = "tie" if total_score == 2: category_5pt = "win" elif total_score == 1: category_5pt = "leaning_win" elif total_score == -1: category_5pt = "leaning_loss" elif total_score == -2: category_5pt = "loss" else: category_5pt = "tie" category_3pt = "tie" if total_score > 0: category_3pt = "win" elif total_score < 0: category_3pt = "loss" else: category_3pt = "tie" result_entry = { "run1": rating1, "run2": rating2, "p1_result_run1": r1_result, "p1_result_run2": r2_result, "total_score": total_score, "category_5pt": category_5pt, "category_3pt": category_3pt, "paper_1_path": paper1_path, "paper_2_path": paper2_path, "exp1_name": exp1_name, "exp2_name": exp2_name, } with FILE_LOCK: current_results_dict[paper_id] = result_entry with open(output_json_path, "w") as f: json.dump(current_results_dict, f, indent=4) print( f"✅ [{paper_id}] Finished rating SxS Lit Review (Double). 5pt: {category_5pt}, 3pt: {category_3pt}", flush=True, ) return paper_id, True def perform_sxs_lit_review_batch( project_folder_1: str, project_folder_2: str, rel_path_1: str, rel_path_2: str, exp1_name: str, exp2_name: str, output_folder: str = "./logs/sxs_lit_review_quality", model_name: str = "gemini-3.1-pro-preview", ): print(f"Starting Enhanced SxS Lit Review batch execution...", flush=True) print(f"Project 1: {project_folder_1} ({rel_path_1})", flush=True) print(f"Project 2: {project_folder_2} ({rel_path_2})", flush=True) print(f"Comparing [{exp1_name}] vs [{exp2_name}] using {model_name}", flush=True) os.makedirs(output_folder, exist_ok=True) conference = "unknown" if "iclr" in project_folder_1.lower(): conference = "iclr" elif "cvpr" in project_folder_1.lower(): conference = "cvpr" output_json_path = os.path.join( output_folder, f"sxs_lit_review_enhanced_{model_name}_{exp1_name}_vs_{exp2_name}_{conference}.json", ) current_results = {} if os.path.exists(output_json_path): with open(output_json_path, "r") as f: current_results = json.load(f) folder1_ids = [ d for d in os.listdir(project_folder_1) if os.path.isdir(os.path.join(project_folder_1, d)) ] folder2_ids = set( [ d for d in os.listdir(project_folder_2) if os.path.isdir(os.path.join(project_folder_2, d)) ] ) common_ids = [] for pid in folder1_ids: if pid in folder2_ids: p1 = os.path.join(project_folder_1, pid, rel_path_1) p2 = os.path.join(project_folder_2, pid, rel_path_2) if os.path.exists(p1) and os.path.exists(p2): if pid not in current_results: common_ids.append((pid, p1, p2)) else: print(f"[{pid}] Already rated. Skipping.", flush=True) print(f"Found {len(common_ids)} overlapping papers to evaluate.", flush=True) max_workers = 3 with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] for pid, p1, p2 in common_ids: f = executor.submit( _process_and_save_single_sxs, pid, p1, p2, output_json_path, current_results, model_name, exp1_name, exp2_name, ) futures.append(f) for future in as_completed(futures): future.result() wins_5pt = 0 leaning_wins_5pt = 0 ties_5pt = 0 leaning_losses_5pt = 0 losses_5pt = 0 wins_3pt = 0 ties_3pt = 0 losses_3pt = 0 valid_evals = 0 for pid, res in current_results.items(): if pid == "_summary_stats": continue if not isinstance(res, dict) or "category_5pt" not in res: continue cat_5 = res["category_5pt"] cat_3 = res["category_3pt"] if cat_5 == "win": wins_5pt += 1 elif cat_5 == "leaning_win": leaning_wins_5pt += 1 elif cat_5 == "tie": ties_5pt += 1 elif cat_5 == "leaning_loss": leaning_losses_5pt += 1 elif cat_5 == "loss": losses_5pt += 1 if cat_3 == "win": wins_3pt += 1 elif cat_3 == "tie": ties_3pt += 1 elif cat_3 == "loss": losses_3pt += 1 valid_evals += 1 stats = { "exp1_name": exp1_name, "exp2_name": exp2_name, "total_valid_evals": valid_evals, "five_point": { "wins": wins_5pt, "leaning_wins": leaning_wins_5pt, "ties": ties_5pt, "leaning_losses": leaning_losses_5pt, "losses": losses_5pt, "win_rate": wins_5pt / valid_evals if valid_evals > 0 else 0, "leaning_win_rate": ( leaning_wins_5pt / valid_evals if valid_evals > 0 else 0 ), "tie_rate": ties_5pt / valid_evals if valid_evals > 0 else 0, }, "three_point": { "wins": wins_3pt, "ties": ties_3pt, "losses": losses_3pt, "win_rate": wins_3pt / valid_evals if valid_evals > 0 else 0, "tie_rate": ties_3pt / valid_evals if valid_evals > 0 else 0, }, } current_results["_summary_stats"] = stats with open(output_json_path, "w") as f: json.dump(current_results, f, indent=4) print("\n" + "=" * 50, flush=True) print("ENHANCED SxS LITERATURE REVIEW QUALITY EVALUATION SUMMARY", flush=True) print("=" * 50, flush=True) print(f"Total Evaluated: {valid_evals}", flush=True) print(f"--- 5-Point Scale ---", flush=True) print( f"[{exp1_name}] Wins: {wins_5pt} ({stats['five_point']['win_rate']:.1%})", flush=True, ) print( f"[{exp1_name}] Leaning Wins: {leaning_wins_5pt} ({stats['five_point']['leaning_win_rate']:.1%})", flush=True, ) print(f"Ties: {ties_5pt} ({stats['five_point']['tie_rate']:.1%})", flush=True) print( f"[{exp2_name}] Leaning Wins (P1 Lean Loss): {leaning_losses_5pt}", flush=True ) print(f"[{exp2_name}] Wins (P1 Loss): {losses_5pt}", flush=True) print(f"--- 3-Point Scale ---", flush=True) print( f"[{exp1_name}] Wins: {wins_3pt} ({stats['three_point']['win_rate']:.1%})", flush=True, ) print(f"Ties: {ties_3pt} ({stats['three_point']['tie_rate']:.1%})", flush=True) print(f"[{exp2_name}] Wins (P1 Loss): {losses_3pt}", flush=True) print("=" * 50 + "\n", flush=True) print(f"Batch completed. Results saved to {output_json_path}", flush=True) - autoraters/sxs_paper_quality.py (14002 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import threading import base64 import time from concurrent.futures import ThreadPoolExecutor, as_completed from google.genai import types import pymupdf # type: ignore from utils.pdf_utils import load_paper from utils.llm_backend_utils import get_llm_parser from utils.gemini_utils import call_gemini_with_contents from utils.openai_utils import call_openai_models_with_content from autoraters.prompts.sxs_quality_prompts import sxs_paper_quality_system_prompt # Global lock for file writing FILE_LOCK = threading.Lock() def get_pdf_page_images_base64(pdf_path, max_pages=None): doc = pymupdf.open(pdf_path) b64_images = [] for i, page in enumerate(doc): if max_pages and i >= max_pages: break # Extract image bytes at screen resolution pix = page.get_pixmap() img_bytes = pix.tobytes("jpeg") b64_str = base64.b64encode(img_bytes).decode("utf-8") b64_images.append(b64_str) return b64_images def rate_sxs_paper_quality( paper1_path: str, paper2_path: str, model_name: str = "gemini-3.1-pro-preview" ): max_retries = 5 cur_try = 0 succeed = False prompt_text = "Task: Compare the overall scientific contribution, technical depth, formatting, presentation, and writing quality of these two papers side by side." is_openai = ( "gpt" in model_name.lower() or "o1" in model_name.lower() or "o3" in model_name.lower() ) while (not succeed) and cur_try < max_retries: try: if is_openai: # Setup OpenAI content payload paper1_text = load_paper(paper1_path, min_size=100) paper2_text = load_paper(paper2_path, min_size=100) content = [] content.append( {"type": "text", "text": "Paper 1 Text:\n" + paper1_text} ) content.append({"type": "text", "text": "Paper 1 Visual Pages:"}) for b64_img in get_pdf_page_images_base64(paper1_path, max_pages=15): content.append( { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, } ) content.append( {"type": "text", "text": "Paper 2 Text:\n" + paper2_text} ) content.append({"type": "text", "text": "Paper 2 Visual Pages:"}) for b64_img in get_pdf_page_images_base64(paper2_path, max_pages=15): content.append( { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, } ) content.append({"type": "text", "text": prompt_text}) response_dict = call_openai_models_with_content( content=content, model_name=model_name, result_parsing_func=get_llm_parser(model_name, return_json=True), check_parsed_response_not_none=True, system_prompt=sxs_paper_quality_system_prompt, ) else: # Setup Gemini with open(paper1_path, "rb") as f1, open(paper2_path, "rb") as f2: pdf1_bytes = f1.read() pdf2_bytes = f2.read() contents = [ types.Part.from_text(text="Paper 1:"), types.Part.from_bytes(data=pdf1_bytes, mime_type="application/pdf"), types.Part.from_text(text="\n\nPaper 2:"), types.Part.from_bytes(data=pdf2_bytes, mime_type="application/pdf"), types.Part.from_text(text="\n\n" + prompt_text), ] response_dict = call_gemini_with_contents( contents=contents, model_name=model_name, result_parsing_func=get_llm_parser(model_name, return_json=True), generation_configs={ "temperature": 0.0, "system_instruction": sxs_paper_quality_system_prompt, }, check_parsed_response_not_none=True, ) succeed = True return ( response_dict.get("parsed_response", response_dict) if isinstance(response_dict, dict) else response_dict ) except Exception as e: print( f"Error processing SxS (model={model_name}): {e}. Retrying [{cur_try + 1}/{max_retries}]...", flush=True, ) cur_try += 1 if cur_try < max_retries: time.sleep(5) return {} def _process_and_save_single_sxs( paper_id: str, paper1_path: str, paper2_path: str, output_json_path: str, current_results_dict: dict, model_name: str, exp1_name: str, exp2_name: str, ): """ Helper function to run inside the thread. Rates SxS paper quality twice (both orders) and aggregates. """ rating1 = rate_sxs_paper_quality(paper1_path, paper2_path, model_name=model_name) rating2 = rate_sxs_paper_quality(paper2_path, paper1_path, model_name=model_name) if not rating1 or not rating2: print( f"❌ [{paper_id}] Failed to get ratings for one or both rounds.", flush=True ) return paper_id, False winner1 = rating1.get("winner", "").lower() if isinstance(rating1, dict) else "" winner2 = rating2.get("winner", "").lower() if isinstance(rating2, dict) else "" r1_result = "tie" if "paper_1" in winner1 or "paper 1" in winner1: r1_result = "p1" elif "paper_2" in winner1 or "paper 2" in winner1: r1_result = "p2" r2_result = "tie" if "paper_1" in winner2 or "paper 1" in winner2: r2_result = "p2" # Swap! elif "paper_2" in winner2 or "paper 2" in winner2: r2_result = "p1" s1 = 1 if r1_result == "p1" else (-1 if r1_result == "p2" else 0) s2 = 1 if r2_result == "p1" else (-1 if r2_result == "p2" else 0) total_score = s1 + s2 category_5pt = "tie" if total_score == 2: category_5pt = "win" elif total_score == 1: category_5pt = "leaning_win" elif total_score == -1: category_5pt = "leaning_loss" elif total_score == -2: category_5pt = "loss" else: category_5pt = "tie" category_3pt = "tie" if total_score > 0: category_3pt = "win" elif total_score < 0: category_3pt = "loss" else: category_3pt = "tie" result_entry = { "run1": rating1, "run2": rating2, "p1_result_run1": r1_result, "p1_result_run2": r2_result, "total_score": total_score, "category_5pt": category_5pt, "category_3pt": category_3pt, "paper_1_path": paper1_path, "paper_2_path": paper2_path, "exp1_name": exp1_name, "exp2_name": exp2_name, } with FILE_LOCK: current_results_dict[paper_id] = result_entry with open(output_json_path, "w") as f: json.dump(current_results_dict, f, indent=4) print( f"✅ [{paper_id}] Finished rating SxS Overall Paper Quality (Double). 5pt: {category_5pt}, 3pt: {category_3pt}", flush=True, ) return paper_id, True def perform_sxs_paper_quality_batch( project_folder_1: str, project_folder_2: str, rel_path_1: str, rel_path_2: str, exp1_name: str, exp2_name: str, output_folder: str = "./logs/sxs_paper_quality", model_name: str = "gemini-3.1-pro-preview", ): print(f"Starting Enhanced SxS Paper Quality batch execution...", flush=True) print(f"Project 1: {project_folder_1} ({rel_path_1})", flush=True) print(f"Project 2: {project_folder_2} ({rel_path_2})", flush=True) print(f"Comparing [{exp1_name}] vs [{exp2_name}] using {model_name}", flush=True) os.makedirs(output_folder, exist_ok=True) conference = "unknown" if "iclr" in project_folder_1.lower(): conference = "iclr" elif "cvpr" in project_folder_1.lower(): conference = "cvpr" output_json_path = os.path.join( output_folder, f"sxs_paper_quality_enhanced_{model_name}_{exp1_name}_vs_{exp2_name}_{conference}.json", ) current_results = {} if os.path.exists(output_json_path): with open(output_json_path, "r") as f: current_results = json.load(f) folder1_ids = [ d for d in os.listdir(project_folder_1) if os.path.isdir(os.path.join(project_folder_1, d)) ] folder2_ids = set( [ d for d in os.listdir(project_folder_2) if os.path.isdir(os.path.join(project_folder_2, d)) ] ) common_ids = [] for pid in folder1_ids: if pid in folder2_ids: p1 = os.path.join(project_folder_1, pid, rel_path_1) p2 = os.path.join(project_folder_2, pid, rel_path_2) if os.path.exists(p1) and os.path.exists(p2): if pid not in current_results: common_ids.append((pid, p1, p2)) else: print(f"[{pid}] Already rated. Skipping.", flush=True) print(f"Found {len(common_ids)} overlapping papers to evaluate.", flush=True) max_workers = 3 # Reduced to avoid rate limits since we do 2 calls per paper with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] for pid, p1, p2 in common_ids: f = executor.submit( _process_and_save_single_sxs, pid, p1, p2, output_json_path, current_results, model_name, exp1_name, exp2_name, ) futures.append(f) for future in as_completed(futures): future.result() wins_5pt = 0 leaning_wins_5pt = 0 ties_5pt = 0 leaning_losses_5pt = 0 losses_5pt = 0 wins_3pt = 0 ties_3pt = 0 losses_3pt = 0 valid_evals = 0 for pid, res in current_results.items(): if pid == "_summary_stats": continue if not isinstance(res, dict) or "category_5pt" not in res: continue cat_5 = res["category_5pt"] cat_3 = res["category_3pt"] if cat_5 == "win": wins_5pt += 1 elif cat_5 == "leaning_win": leaning_wins_5pt += 1 elif cat_5 == "tie": ties_5pt += 1 elif cat_5 == "leaning_loss": leaning_losses_5pt += 1 elif cat_5 == "loss": losses_5pt += 1 if cat_3 == "win": wins_3pt += 1 elif cat_3 == "tie": ties_3pt += 1 elif cat_3 == "loss": losses_3pt += 1 valid_evals += 1 stats = { "exp1_name": exp1_name, "exp2_name": exp2_name, "total_valid_evals": valid_evals, "five_point": { "wins": wins_5pt, "leaning_wins": leaning_wins_5pt, "ties": ties_5pt, "leaning_losses": leaning_losses_5pt, "losses": losses_5pt, "win_rate": wins_5pt / valid_evals if valid_evals > 0 else 0, "leaning_win_rate": ( leaning_wins_5pt / valid_evals if valid_evals > 0 else 0 ), "tie_rate": ties_5pt / valid_evals if valid_evals > 0 else 0, }, "three_point": { "wins": wins_3pt, "ties": ties_3pt, "losses": losses_3pt, "win_rate": wins_3pt / valid_evals if valid_evals > 0 else 0, "tie_rate": ties_3pt / valid_evals if valid_evals > 0 else 0, }, } current_results["_summary_stats"] = stats with open(output_json_path, "w") as f: json.dump(current_results, f, indent=4) print("\n" + "=" * 50, flush=True) print("ENHANCED SxS PAPER QUALITY EVALUATION SUMMARY", flush=True) print("=" * 50, flush=True) print(f"Total Evaluated: {valid_evals}", flush=True) print(f"--- 5-Point Scale ---", flush=True) print( f"[{exp1_name}] Wins: {wins_5pt} ({stats['five_point']['win_rate']:.1%})", flush=True, ) print( f"[{exp1_name}] Leaning Wins: {leaning_wins_5pt} ({stats['five_point']['leaning_win_rate']:.1%})", flush=True, ) print(f"Ties: {ties_5pt} ({stats['five_point']['tie_rate']:.1%})", flush=True) print( f"[{exp2_name}] Leaning Wins (P1 Lean Loss): {leaning_losses_5pt}", flush=True ) print(f"[{exp2_name}] Wins (P1 Loss): {losses_5pt}", flush=True) print(f"--- 3-Point Scale ---", flush=True) print( f"[{exp1_name}] Wins: {wins_3pt} ({stats['three_point']['win_rate']:.1%})", flush=True, ) print(f"Ties: {ties_3pt} ({stats['three_point']['tie_rate']:.1%})", flush=True) print(f"[{exp2_name}] Wins (P1 Loss): {losses_3pt}", flush=True) print("=" * 50 + "\n", flush=True) print(f"Batch completed. Results saved to {output_json_path}", flush=True) - CODE_OF_CONDUCT.md (4487 bytes)
# Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. This Code of Conduct also applies outside the project spaces when the Project Steward has a reasonable belief that an individual's behavior may have a negative impact on the project or its community. ## Conflict Resolution We do not believe that all conflict is bad; healthy debate and disagreement often yield positive results. However, it is never okay to be disrespectful or to engage in behavior that violates the project’s code of conduct. If you see someone violating the code of conduct, you are encouraged to address the behavior directly with those involved. Many issues can be resolved quickly and easily, and this gives people more control over the outcome of their dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe. Reports should be directed to *Yiwen Song ([email protected])*, the Project Steward for PaperOrchestra. It is the Project Steward’s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you are uncomfortable reaching out to the Project Steward, please email [email protected]. We will investigate every complaint, but you may not receive a direct response. We will use our discretion in determining when and how to follow up on reported incidents, which may range from not taking action to permanent expulsion from the project and project-sponsored spaces. We will notify the accused of the report and provide them an opportunity to discuss it before any action is taken. The identity of the reporter will be omitted from the details of the report supplied to the accused. In potentially harmful situations, such as ongoing harassment or threats to anyone's safety, we may take action without notice. ## Attribution This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct - CONTRIBUTING.md (1128 bytes)
# How to contribute We'd love to accept your patches and contributions to this project. ## Before you begin ### Sign our Contributor License Agreement Contributions to this project must be accompanied by a [Contributor License Agreement](https://cla.developers.google.com/about) (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. If you or your current employer have already signed the Google CLA (even if it was for a different project), you probably don't need to do it again. Visit <https://cla.developers.google.com/> to see your current agreements or to sign a new one. ### Review our community guidelines This project follows [Google's Open Source Community Guidelines](https://opensource.google/conduct/). ## Contribution process ### Code reviews All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more information on using pull requests.
- frontend/app.py (46724 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import streamlit as st import os import json import sys import time import threading import smtplib from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from email import encoders from dotenv import load_dotenv import subprocess import shutil import tempfile import datetime import io import zipfile from streamlit.runtime.scriptrunner import add_script_run_ctx import signal import ctypes from frontend_utils import send_notification_email load_dotenv() sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from methods.paper_writer import write_single_paper from utils.pdf_utils import pdf_to_grid_images st.set_page_config( page_title="Paper Orchestra - AI Paper Writer", page_icon="🎻", layout="wide", initial_sidebar_state="expanded", ) def apply_custom_css(): st.markdown( """ <style> .stApp { background: #f8fafc; color: #0f172a; } [data-testid="stSidebar"] { background-color: #f1f5f9; border-right: 1px solid #e2e8f0; } .card { background: white; border-radius: 12px; padding: 2rem; border: 1px solid #e2e8f0; box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.1), 0 1px 2px 0 rgba(0, 0, 0, 0.06); margin-bottom: 1.5rem; } .timeline-container { padding: 1rem; } .timeline-item { border-left: 2px solid #e2e8f0; padding-left: 1.5rem; position: relative; margin-bottom: 1.5rem; } .timeline-item.active { border-left-color: #2563eb; } .timeline-item.finished { border-left-color: #10b981; } .timeline-icon { position: absolute; left: -11px; top: 0; width: 20px; height: 20px; border-radius: 50%; background: white; border: 2px solid #cbd5e1; display: flex; align-items: center; justify-content: center; font-size: 0.75rem; color: #64748b; } .timeline-item.active .timeline-icon { border-color: #b45309; background: #fffbeb; color: #b45309; } .timeline-item.finished .timeline-icon { border-color: #10b981; background: #10b981; color: white; } .gallery-item { background: white; border-radius: 8px; padding: 1rem; text-align: center; border: 1px solid #e2e8f0; box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); } .stButton > button { background-color: #ecfdf5 !important; color: #065f46 !important; border: 1px solid #a7f3d0 !important; padding: 0.75rem 1.5rem; border-radius: 6px; font-weight: 600; width: 100%; transition: all 0.2s; } .stButton > button:hover { transform: translateY(-1px); background-color: #d1fae5 !important; box-shadow: 0 4px 6px -1px rgba(5, 150, 105, 0.2); } .stTextInput > div > div > input, .stTextArea > div > div > textarea { background-color: white !important; color: #0f172a !important; border: 1px solid #cbd5e1 !important; } </style> """, unsafe_allow_html=True, ) apply_custom_css() # Session State Initialization def initialize_session_state(): default_states = { "locked": False, "running": False, "progress": 0, "hide_inputs": False, "logs": { "Outline Generation": "", "Literature Review": "", "Figure Plotting": "", "Section Writing": "", "Content Refinement": "", }, "current_step": 0, "figures": [], "figure_counter": 0, "generated_figures": [], "run_directory": None, "email_confirmed": False, "error_message": "", } for state_key, default_value in default_states.items(): if state_key not in st.session_state: st.session_state[state_key] = default_value initialize_session_state() def render_sidebar(): st.sidebar.title("🎻 Configuration") if st.sidebar.button("Load CVPR Example", disabled=st.session_state.locked): examples_directory = os.path.join( os.path.dirname(os.path.abspath(__file__)), "examples" ) javascript_object_notation_path = os.path.join( examples_directory, "cvpr_example.json" ) try: with open(javascript_object_notation_path, "r") as file_object: example_data = json.load(file_object) st.session_state.idea_text_main = example_data.get("idea", "") st.session_state.log_text_main = example_data.get("log", "") figure_path = example_data.get("figure_path") figure_caption = example_data.get("figure_caption", "") if figure_path: full_figure_path = os.path.join(examples_directory, figure_path) if os.path.exists(full_figure_path): st.session_state.figures = [ { "file": full_figure_path, "name": figure_path, "caption": figure_caption, } ] st.sidebar.success("🎉 Example loaded successfully!") time.sleep(1) st.rerun() except Exception as exception_object: st.sidebar.error(f"Error loading example: {exception_object}") models = ["gemini-3.1-pro-preview", "gemini-3-flash-preview"] selected_model = st.sidebar.selectbox( "Select Model", models, disabled=st.session_state.locked ) templates = ["cvpr2025", "iclr2025"] selected_template = st.sidebar.selectbox( "Select Template", templates, disabled=st.session_state.locked ) plotting_options = ["Yes", "No"] enable_plotting = st.sidebar.selectbox( "Enable Figure Generation", plotting_options, index=0, disabled=st.session_state.locked, ) st.sidebar.markdown( "Your Email (for notification) <span style='color:red'>*</span>", unsafe_allow_html=True, ) email_input = st.sidebar.text_input( "Email", key="email_input", label_visibility="collapsed", disabled=st.session_state.locked, ) if st.sidebar.button("Confirm Email", disabled=st.session_state.locked): if st.session_state.get("email_input"): st.session_state.email_confirmed = True else: st.session_state.email_confirmed = False st.sidebar.warning("⚠️ Please enter an email address first.") if st.session_state.get("email_confirmed"): st.sidebar.success("📧 Email confirmed successfully!") return selected_model, selected_template, enable_plotting, email_input selected_model, selected_template, enable_plotting, email_input = render_sidebar() # ------------------------------------------------------------------------ # UI COMPONENT FUNCTIONS # ------------------------------------------------------------------------ def render_gallery_tab(): st.subheader("Uploaded Figures") if not st.session_state.figures: if not st.session_state.running: st.markdown( "<p style='color: gray; font-style: italic;'>No figures uploaded yet. Add figures from the input materials section.</p>", unsafe_allow_html=True, ) else: st.info("ℹ️ No user figures provided for this run.") else: for index, figure in enumerate(st.session_state.figures): st.markdown( f'<div class="gallery-item" style="margin-bottom: 1rem;"><p style="font-weight: bold;">{figure["name"]}</p></div>', unsafe_allow_html=True, ) st.image(figure["file"], width="stretch") new_caption = st.text_input( "Edit Caption", value=figure["caption"], key=f"edit_caption_{index}", disabled=st.session_state.locked, ) if new_caption != figure["caption"]: st.session_state.figures[index]["caption"] = new_caption if st.button( "🗑️ Delete Image", key=f"delete_figure_{index}", disabled=st.session_state.locked, ): st.session_state.figures.pop(index) st.rerun() st.markdown("<hr>", unsafe_allow_html=True) st.subheader("Generated Figures") if st.session_state.get("run_directory"): results_path = os.path.join( st.session_state.run_directory, "plotting_results.json" ) if os.path.exists(results_path): try: with open(results_path, "r") as file_object: st.session_state.generated_figures = json.load(file_object) except Exception: pass if st.session_state.get("generated_figures"): for figure in st.session_state.generated_figures: image_path = os.path.join( st.session_state.run_directory, figure.get("image_path", "") ) if os.path.exists(image_path): st.markdown(f"**{figure.get('title', 'Figure')}**") st.image(image_path, caption=figure.get("caption", "")) st.markdown("<hr>", unsafe_allow_html=True) def render_json_file(file_name, title, failure_message): st.subheader(title) if st.session_state.get("run_directory"): file_path = os.path.join(st.session_state.run_directory, file_name) if os.path.exists(file_path): try: with open(file_path, "r") as file_object: st.json(json.load(file_object)) except Exception as exception_object: st.error(f"Error loading {title.lower()}: {exception_object}") elif not st.session_state.running: st.info(failure_message) else: st.info(failure_message) def render_pdf_and_snapshots(pdf_path, snapshots_directory, download_key): if os.path.exists(pdf_path): with open(pdf_path, "rb") as file_object: st.download_button( label="📥 Download PDF", data=file_object.read(), file_name=os.path.basename(pdf_path), mime="application/pdf", key=download_key, ) if os.path.exists(snapshots_directory): st.markdown("**Snapshots:**") image_files = sorted( [ file for file in os.listdir(snapshots_directory) if file.endswith(".jpg") or file.endswith(".png") ] ) if image_files: grid_columns = st.columns(3) for index, image_file in enumerate(image_files[:3]): with grid_columns[index]: st.image( os.path.join(snapshots_directory, image_file), width="stretch", ) # ------------------------------------------------------------------------ # BACKGROUND EXECUTION POLLING # ------------------------------------------------------------------------ def handle_live_execution( steps, email_input, selected_template, selected_model, enable_plotting ): if st.session_state.running: if not st.session_state.get("thread_started"): if ( not st.session_state.get("saved_idea") or not st.session_state.get("saved_log") or not st.session_state.get("email_input") ): st.session_state.error_message = ( "❌ Please fill in all required fields (Idea, Log, and Email)." ) st.session_state.running = False st.session_state.locked = False st.session_state.hide_inputs = ( False # Fixed: ensures inputs pop back up if validation fails ) st.rerun() st.session_state.thread_started = True base_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parent_directory = os.path.dirname(base_directory) timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") run_directory = os.path.join( parent_directory, "demo_logs", f"paper_orchestra_run_{timestamp}" ) st.session_state.run_directory = run_directory raw_materials_directory = os.path.join(run_directory, "raw_materials") figures_directory = os.path.join(raw_materials_directory, "figures") os.makedirs(run_directory, exist_ok=True) os.makedirs(raw_materials_directory, exist_ok=True) os.makedirs(figures_directory, exist_ok=True) command = [ sys.executable, "-u", os.path.join(base_directory, "paper_writing_cli.py"), "--raw_materials_dir", raw_materials_directory, "--output_dir", run_directory, "--latex_template_dir", os.path.join(base_directory, "templates", selected_template), "--writer_model_name", selected_model, "--reflection_model_name", selected_model, "--idea_filename", "idea_sparse.md", "--experimental_log_filename", "experimental_log.md", ] if enable_plotting == "Yes": command.extend( [ "--use_plotting", "True", "--plotting_model_name", selected_model, "--image_model_name", "gemini-3-pro-image-preview", ] ) def run_pipeline( execution_command, working_directory, steps_list, recipient_email, idea_content, log_content, figures_list, ): try: libc = ctypes.CDLL(None) process_set_parent_death_signal = 1 def set_death_signal(): libc.prctl(process_set_parent_death_signal, signal.SIGKILL) os.makedirs(run_directory, exist_ok=True) - frontend/examples/cvpr_example.json (13628 bytes)
{ "__comment__": "This example contains raw materials extracted from [this paper](https://openaccess.thecvf.com/content/CVPR2025/papers/Radman_TSAM_Temporal_SAM_Augmented_with_Multimodal_Prompts_for_Referring_Audio-Visual_CVPR_2025_paper.pdf).", "idea": "## Problem Statement\nThe Segment Anything Model (SAM) has established a new baseline for static image segmentation; however, it is structurally ill-equipped for Referring Audio-Visual Segmentation (Ref-AVS). Current foundation models like SAM suffer from two critical limitations in this context: \n1. **Lack of Temporal Awareness:** SAM processes inputs as isolated static frames, failing to capture the temporal consistency and dynamic context necessary for video segmentation.\n2. **Reliance on Explicit Interaction:** SAM depends on manual user prompts (points, boxes, or masks) to identify targets. It lacks the native ability to interpret implicit \"multimodal prompts\"\u2014such as identifying an object described by a specific sound or textual description\u2014without human intervention.\n\n## Core Hypothesis\nWe hypothesize that we can adapt the frozen, pre-trained knowledge of SAM for dynamic audio-visual scenes without heavy retraining by introducing two novel architectural components. First, a parallel **temporal modeling branch** can inject time-series context into the static encoder. Second, we can replace the manual \"user-in-the-loop\" prompting mechanism with **data-driven multimodal prompts**. By translating aligned audio and text features into the sparse and dense prompt formats that SAM natively understands, we believe the model can learn to \"prompt itself\" to segment temporally evolving objects described by complex semantic cues.\n\n## Proposed Methodology (High-Level Technical Approach)\n\nWe propose **TSAM**, a modified architecture that wraps the standard SAM backbone. Our approach focuses on minimal trainable additions while keeping the heavy image encoder frozen.\n\n**1. Temporal Modeling Branch (Context Injection)**\nInstead of retraining the heavyweight image encoder, we will introduce a lightweight auxiliary branch running in parallel.\n* **Sequential Processing:** As the frozen encoder processes a frame, this parallel branch will accept features from current and previous steps.\n* **Cached Memory & Adapters:** We will utilize a memory mechanism to store a running summary of previous frame representations. An adapter module will fuse these historical visual states with current audio-text embeddings. This allows the model to maintain object permanence and consistency across time.\n\n**2. Automated Multimodal Prompting**\nWe aim to synthesize the \"prompts\" SAM expects (points and masks) using audio-visual-text correlations rather than manual clicks.\n* **Sparse Prompting Module (Simulating Points):** We will design a query selection mechanism. By analyzing the correlation between the reference text and the audio stream, the system will select the most relevant \"audio cues.\" These cues will be projected and treated as sparse queries (analogous to point prompts) to guide the mask decoder toward the sound source.\n* **Dense Prompting Module (Simulating Masks):** We will implement a deep fusion module that uses cross-attention to mix visual features with aligned audio and text embeddings. This will generate a dense, spatial feature map that acts as a coarse \"mask prompt,\" providing global context to the decoder.\n\n**3. Decoding**\nThe final segmentation will be generated by the standard SAM mask decoder, which will be queried by our synthetic sparse and dense prompts, refined by the temporally aware features from our auxiliary branch.\n\n## Expected Contribution\n* **Architectural Novelty:** A framework for extending static, unimodal foundation models (like SAM) into the temporal and multimodal domains without compromising their zero-shot capabilities.\n* **Methodological Advancement:** A demonstration of \"latent prompting,\" where semantic cues (audio/text) are mathematically translated into the geometric prompts (points/masks) required by segmentation models.\n* **Theoretical Impact:** Establishing that lightweight temporal adapters and cross-modal attention mechanisms are sufficient to unlock video understanding in static image backbones.", "log": "## 1. Experimental Setup\n\nWe conducted a comprehensive evaluation of the proposed TSAM method for the Referring Audio-Visual Segmentation (Ref-AVS) task.\n\n* **Datasets:**\n * **Ref-AVS Dataset:** We utilized the Ref-AVS dataset containing 20,000 text expressions and pixel-level annotations across 4,000 10-second videos.\n * **Object Categories:** The dataset included audible objects (20 musical instruments, 8 animals, 15 machines, 5 humans) and 3 categories of static, inaudible objects.\n * **Splits:**\n * **Training Set:** 2,908 videos.\n * **Validation Set:** 276 videos.\n * **Test Set:** 818 videos total, subdivided into:\n * *Seen:* 292 videos (categories present in training).\n * *Unseen:* 269 videos (categories not seen during training; tests generalization).\n * *Null:* 257 videos (text refers to non-existent/not visible objects; empty true masks).\n\n* **Evaluation Metrics:**\n * **Standard Metrics:** We employed the Jaccard Index ($_mathcal{J}$) and F-score ($_mathcal{F}$) for the Seen and Unseen subsets.\n * **Null Subset Metric:** We employed the metric $_mathcal{S}$, which measures the ratio of the predicted mask area to the background area ($_mathcal{S} = \\sqrt{\text{predicted mask area} / \text{background area}}$). A lower $_mathcal{S}$ value indicates better performance (less incorrect segmentation).\n\n Baselines Compared:**\n * **AVS Methods (Augmented with Text):** AVSBench (PVT-v2 backbone), AVSegFormer (PVT-v2 backbone), GAVS (SAM backbone), SAMA (SAM backbone). *Note: We re-evaluated SAMA by integrating text with audio/visual inputs.*\n * **Ref-VOS Methods (Augmented with Audio):** ReferFormer (Video-Swin backbone), R2-VOS (Video-Swin backbone).\n * **Ref-AVS Method:** EEMC (Mask2Former backbone; previous state-of-the-art).\n\n* **Implementation Details:**\n * **Model Initialization:** We initialized TSAM using the pre-trained ViT-B variant of SAM ($N=12$, embedding dimension $d_{emb}=256$).\n * **Architecture Settings:** The temporal branch consisted of $M=4$ blocks. The number of selected audio queries was set to $k=5$.\n * **Hardware:** Training was performed on eight AMD GPUs in a distributed setup.\n * **Optimizer:** AdamW optimizer was used.\n * **Hyperparameters:** Initial learning rate of $1 \\cdot 10^{-4}$, batch size of 1.\n * **Training Duration:** The model was trained for 15 epochs with periodic evaluations.\n * **Model Selection:** We selected the best-performing model on the validation subset for testing.\n\n## 2. Raw Numeric Data\n\n**Table 1: Performance comparison on the Ref-AVS dataset**\n*Note: Baselines marked with \u2020 had text integration added; baselines marked with \u2021 had audio integration added; * marks our re-implementation of SAMA with text added.*\n\n| Method | Task | Visual Backbone | Seen J(%) | Seen F | Unseen J(%) | Unseen F | Null S(\u2193) |\n| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n| AVSBench | AVS\u2020 | PVT-v2 | 23.20 | 0.511 | 32.36 | 0.547 | 0.208 |\n| AVSegFormer | AVS\u2020 | PVT-v2 | 33.47 | 0.470 | 36.05 | 0.501 | 0.171 |\n| GAVS | AVS\u2020 | SAM | 28.93 | 0.498 | 29.82 | 0.497 | 0.190 |\n| SAMA | AVS* | SAM | 39.22 | 0.562 | 47.50 | 0.566 | 0.130 |\n| ReferFormer | Ref-VOS\u2021 | V-Swin | 31.31 | 0.501 | 30.40 | 0.488 | 0.176 |\n| R2-VOS | Ref-VOS\u2021 | V-Swin | 25.01 | 0.410 | 27.93 | 0.498 | 0.183 |\n| EEMC | Ref-AVS | M2F | 34.20 | 0.513 | 49.54 | 0.648 | **0.007** |\n| **TSAM (Ours)** | Ref-AVS | SAM | **43.43** | **0.568** | **54.58** | **0.664** | 0.017 |\n\n**Table 2: Ablation study on TSAM components and IoU loss**\n*Legend: TB=Temporal Branch, TMFL=Temporal Modality Fusion Layer, DPM=Dense Prompting Module, SPM=Sparse Prompting Module, CM=Cached Memory, AM=Adapter Module. Mix(S+U) is the average of Seen and Unseen.*\n\n| Setting | Seen J(%) | Seen F | Unseen J(%) | Unseen F | Mix(S+U) J(%) | Mix(S+U) F | Null S(\u2193) |\n| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n| \u2460 TSAM (Full) | **43.43** | 0.568 | **54.58** | **0.664** | **49.01** | **0.616** | 0.017 |\n| \u2461 - TB | 33.05 | 0.507 | 50.48 | 0.657 | 41.77 | 0.582 | 0.505 |\n| \u2462 - TMFL | 40.35 | 0.579 | 45.54 | 0.627 | 42.95 | 0.603 | 0.018 |\n| \u2463 - DPM | 42.72 | 0.580 | 49.10 | 0.647 | 45.91 | 0.614 | 0.018 |\n| \u2464 - SPM | 43.04 | 0.580 | 49.75 | 0.652 | 46.40 | **0.616** | 0.018 |\n| \u2465 - SPM+DPM | 42.60 | 0.602 | 40.58 | 0.604 | 41.59 | 0.603 | 0.018 |\n| \u2466 - CM(a+v) | 42.07 | 0.544 | 49.11 | 0.659 | 45.59 | 0.602 | 0.018 |\n| \u2467 - CM(v) | 42.75 | 0.549 | 51.18 | 0.660 | 46.97 | 0.605 | 0.018 |\n| \u2468 - AM | 43.13 | **0.600** | 40.79 | 0.617 | 41.96 | 0.609 | 0.017 |\n| \u2469 - $_mathcal{L}_{IoU}$ | 38.29 | 0.564 | 42.15 | 0.631 | 40.22 | 0.598 | **0.008** |\n\n**Table 3: Effect of audio queries ($k$) and temporal branch depth ($M$)**\n\n| Variation | Seen J(%) | Seen F | Unseen J(%) | Unseen F | Mix(S+U) J(%) | Mix(S+U) F | Null S(\u2193) |\n| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n| **$k=3$** | 43.58 | **0.579** | 50.43 | 0.655 | 47.01 | **0.617** | 0.018 |\n| **$k=5$** | 43.43 | 0.568 | **54.58** | **0.664** | **49.01** | 0.616 | **0.017** |\n| **$k=7$** | 43.24 | 0.573 | 46.95 | 0.630 | 45.10 | 0.601 | 0.018 |\n| **$M=2$** | 42.04 | 0.566 | 53.51 | 0.651 | 47.76 | 0.609 | 0.023 |\n| **$M=4$** | **43.43** | 0.568 | **54.58** | **0.664** | **49.01** | **0.616** | **0.017** |\n| **$M=6$** | 43.27 | **0.575** | 49.86 | 0.640 | 46.57 | 0.608 | 0.020 |\n\n## 3. Qualitative Observations\n\n**Comparisons with State-of-the-Art:**\n* **Backbone Analysis:** We observed that methods utilizing prior segmentation visual backbones (SAM and Mask2Former) generally outperformed those based on PVT-v2 and V-Swin backbones.\n* **SAM-Based Baseline Limitations:** Although GAVS and SAMA are SAM-based, they performed worse than EEMC. We noted that SAMA failed to leverage SAM's flexible, promptable nature, and GAVS lacked cohesive multimodal fusion.\n* **TSAM Performance:** TSAM achieved the highest performance on both Seen and Unseen test sets. Specifically, TSAM improved over EEMC by 9.23% in Jaccard Index on the Seen set and 5.04% on the Unseen set.\n* **Null Set Performance:** TSAM fell slightly behind EEMC on the Null test set (S value 0.017 vs 0.007). We attribute this to SAM's inherent limitation of always attempting to produce a segmentation mask even when no target object is present.\n\n**Ablation Observations:**\n* **Temporal Branch (TB):** Removing the temporal branch caused the most significant performance degradation (Seen Jaccard dropped from 43.43% to 33.05%). This confirmed the critical role of temporal modeling for generalizing across video frames.\n* **Prompting Modules (SPM/DPM):** We found that the Sparse Prompting Module (SPM) and Dense Prompting Module (DPM) play complementary roles. Removing both simultaneously resulted in a notable decrease in segmentation performance.\n* **Cached Memory (CM):** Omitting cached memory, especially the visual-only memory, degraded performance, highlighting the importance of shared memory for temporal alignment.\n* **Adapter Module (AM):** The omission of the adapter module caused a significant performance drop, particularly on the Unseen test set, validating its role in facilitating deep multimodal interaction.\n* **Loss Function:** Including the IoU loss ($_mathcal{L}_{IoU}$) improved performance on Seen and Unseen sets but slightly degraded the Null score.\n\n**Hyperparameter Sensitivity:**\n* **Audio Queries ($k$):** We found that $k=5$ yielded optimal results. A lower value ($k=3$) performed well on Seen data but worse on Unseen, while a higher value ($k=7$) likely introduced irrelevant queries, overloading the decoder.\n* **Temporal Depth ($M$):** $M=4$ blocks provided the best balance. Shallower setups ($M=2$) lacked sufficient temporal depth, while deeper setups ($M=6$) appeared to introduce excessive complexity that impaired generalization.\n\n**Visual Qualitative Analysis:**\n* **Seen Test Set:** We observed that TSAM consistently produced high-quality masks for targeted objects. It successfully segmented inaudible objects when guided by textual cues (e.g., \"The object behind the sounding women\"), demonstrating effective processing of complex multimodal instructions.\n* **Unseen Test Set:** TSAM demonstrated a remarkable capacity to generalize to novel objects. It effectively aligned audio-visual and textual inputs in novel scenes (e.g., segmenting a \"truck moving\" or \"tuba being played\" that were not in training categories).\n* **Generalization:** The visual results underscored TSAM's ability to preserve SAM's pre-trained knowledge while using the temporal branch to understand dynamic scenes.", "figure_path": "cvpr_example_figure.png", "figure_caption": "Overview of TSAM. Left: TSAM integrates multimodal cues from diverse inputs, leveraging the prompting modules to guide segmentation. Right: The temporal branch fuses multimodal cues and learns temporal context for segmentation in audio-visual scenes. The adapter input, F̂a, is the output from the cached memory." } - frontend/frontend_utils.py (2609 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase from email import encoders from dotenv import load_dotenv load_dotenv() def send_notification_email(recipient_email, paper_name, attachments=None): sender_email = os.getenv("SMTP_EMAIL") sender_password = os.getenv("SMTP_PASSWORD") if not sender_password: return False, "SMTP password not configured." try: msg = MIMEMultipart() msg["From"] = sender_email msg["To"] = recipient_email msg["Subject"] = f"Paper Orchestra: {paper_name} is Complete!" body = f""" <html> <body> <h2>🎻 Paper Orchestra Notification</h2> <p>Great news! Your paper has been successfully generated.</p> <p>Check the attachments for the final PDF and LaTeX source.</p> <hr> <p><small>Generated by Paper Orchestra AI</small></p> </body> </html> """ msg.attach(MIMEText(body, "html")) if attachments: for file_path in attachments: if os.path.exists(file_path): with open(file_path, "rb") as f: part = MIMEBase("application", "octet-stream") part.set_payload(f.read()) encoders.encode_base64(part) part.add_header( "Content-Disposition", f'attachment; filename="{os.path.basename(file_path)}"', ) msg.attach(part) else: print(f" >> Warning: Attachment file not found: {file_path}") with smtplib.SMTP_SSL("smtp.gmail.com", 465) as server: server.login(sender_email, sender_password) server.send_message(msg) return True, "Email sent successfully!" except Exception as e: return False, str(e) - frontend/README.md (1288 bytes)
# Paper Orchestra Frontend This directory contains the Streamlit frontend for Paper Orchestra. ## How to Launch the Demo Locally 1. Ensure you have activated your environment and installed dependencies. 2. Navigate to the project root directory: ```bash cd paper-orchestra conda activate paper_orchestra ``` 3. Run the Streamlit application: ```bash streamlit run frontend/app.py ``` 4. Open the URL provided in the terminal (usually `http://localhost:8501` or similar) in your browser. ## How to Kill Unfinished Writing Processes The frontend launches a background process (`paper_writing_cli.py`) to handle the paper generation. ### Automatic Cleanup On Linux, the application is configured to automatically kill the background process if the Streamlit server process is stopped. Simply pressing **Ctrl+C** in the terminal where you ran `streamlit run` should clean up the running pipeline. ### Manual Cleanup If you disconnect from the frontend or the server stops unexpectedly without cleaning up, you may have orphan processes running in the background. You can kill them manually using: ```bash pkill -f paper_writing_cli.py ``` Or to find the process ID (PID) first: ```bash pgrep -f paper_writing_cli.py ``` And then kill it: ```bash kill <PID> ```
- INSTRUCTIONS.md (515 bytes)
You are the autonomous runtime coordinator for paper-orchestra. System Archetype: multi_agent. Description: PaperOrchestra is a multi-agent framework for automated AI research paper writing. Core Directives: 1. Execute operations deterministically following verified repository manifests. 2. Strictly enforce safe input validation and local-first execution boundaries. 3. Provide structured outputs, formatted logs, and actionable resolution steps. 4. Preserve data integrity and respect local workspace isolation.
- LICENSE (11357 bytes)
Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - methods/agents/content_refinement_agent.py (23036 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import traceback import json import shutil from google.genai import types # type: ignore from utils.pdf_utils import compile_latex, load_paper, pdf_to_grid_images from utils.gemini_utils import ( call_gemini_with_contents, parse_gemini_json_latex_response, call_gemini_with_images, ) from autoraters.agent_review import perform_review_agentreview from methods.prompts.content_refinement_agent import ( content_refinement_agent_system_prompt, ) class ContentRefinementAgent: def __init__( self, experimental_log_path: str, citation_map_path: str, guidelines_path: str, model_name: str = "gemini-3.1-pro-preview", max_reflections: int = 3, work_dir: str = "./paper_workspace", ): """ Agent responsible for refining a paper by dynamically generating reviews and addressing feedback in a loop. """ self.model_name = model_name self.max_reflections = max_reflections self.system_prompt = content_refinement_agent_system_prompt self.work_dir = work_dir os.makedirs(self.work_dir, exist_ok=True) os.makedirs( os.path.join(self.work_dir, "pdf_screenshots", "content_refinement"), exist_ok=True, ) os.makedirs( os.path.join(self.work_dir, "pdf_screenshots", "format_refinement"), exist_ok=True, ) os.makedirs(os.path.join(self.work_dir, "peer_reviews"), exist_ok=True) print(f" >> Work directory: {self.work_dir}") self.experimental_log_path = experimental_log_path self.guidelines_path = guidelines_path self.citation_map_path = citation_map_path self.worklogs = {} self.format_worklogs = {} self.current_tex = "" self.initial_score = 0 self.current_score = 0 def _read_file(self, path: str) -> str: try: with open(path, "r", encoding="utf-8") as f: return f.read() except FileNotFoundError: print(f" >> Warning: File not found at {path}") return "" def _get_peer_review(self, pdf_path: str, iteration_label: str) -> dict: """Helper to call the external reviewer.""" print(f" >> [AgentReview] Generating feedback for {iteration_label} paper...") try: paper_text = load_paper(pdf_path=pdf_path) review = perform_review_agentreview( text=paper_text, model_name=self.model_name, ) score = review.get("Overall", 0) print(f" >> [Reviewer] Score for {iteration_label}: {score}/10") version_name = "v0" if iteration_label == "initial" else iteration_label review_path = os.path.join( self.work_dir, "peer_reviews", f"review_{version_name}.json" ) with open(review_path, "w", encoding="utf-8") as f: json.dump(review, f, indent=4) return review, float(score) except Exception as e: print(f" >> [Reviewer] Error generating review: {e}") traceback.print_exc() return {"Error": str(e), "Overall": 0}, 0.0 def _get_formatting_review(self, pdf_path: str, iteration_label: str) -> dict: """Helper to call VLM for formatting feedback.""" print( f" >> [VLM] Generating formatting feedback for {iteration_label} paper..." ) try: v_num = iteration_label.split("_")[-1] screenshots_dir = os.path.join( self.work_dir, "pdf_screenshots", "format_refinement", f"v{v_num}" ) image_paths = pdf_to_grid_images(pdf_path, screenshots_dir) guidelines = self._read_file(self.guidelines_path) prompt = ( """ Analyze the provided page images of a research paper for formatting issues. You must strictly follow the provided formatting guidelines and examples. Do not create rules based on your own assumptions or general knowledge if they are not explicitly supported by the guidelines or contradicted by the examples. You must output the status for EVERY SINGLE figure and table found in the paper, and then list any other text content or layout issues. Pay attention to column margins and page edges. Only report issues if a figure or table clearly overflows into the gutter or margin. Do not flag figures or tables that are simply filling the column width fully. Formatting Guidelines: """ + guidelines + """ Respond in JSON format with the following structure: ```json { "figure_and_tables": { "Figure 1": { "detected_issue": "The figure is too wide and spills into the right margin.", "suggested_fix": "Use `[width=\\\\linewidth]` in `\\\\includegraphics` to scale the figure to the column width." }, "Table 1": { "detected_issue": "None", "suggested_fix": "None" }, # ... }, "other_issues": [ { "page": "Integer (page number starting from 1)", "element": "String ('Section x, Paragraph y' or other specific location details)", "detected_issue": "String describing the layout or text content issue", "suggested_fix": "String suggesting the fix" }, # ... ] } ``` """ ) response_dict = call_gemini_with_images( prompt=prompt, images=image_paths, model_name=self.model_name, ) formatting_review = response_dict["parsed_response"] return formatting_review except Exception as e: print(f" >> [VLM] Error generating formatting review: {e}") traceback.print_exc() return {"Error": str(e)} def _extract_scores(self, review: dict) -> dict: """Extracts numerical scores from the review dictionary for comparison.""" axes = [ "Originality", "Quality", "Clarity", "Significance", "Soundness", "Presentation", "Contribution", "Overall", "Confidence", ] scores = {} for axis in axes: val = review.get(axis, 0) try: scores[axis] = float(val) except ValueError: scores[axis] = 0.0 return scores def run(self, texfile_path: str) -> str: """ Main execution loop: 1. Compile Initial Draft -> Review. 2. Refine -> Compile -> Review. 3. Repeat until score increases or max_reflections reached. """ self.current_tex = self._read_file(texfile_path) worklog_save_path = os.path.join( self.work_dir, "content_refinement_worklog.json" ) context_files = { "guidelines": self._read_file(self.guidelines_path), "experimental_log": self._read_file(self.experimental_log_path), "citations": self._read_file(self.citation_map_path), } # --- STEP 0: Initial Compilation & Baseline Review --- print("\n=== Initial Baseline Review ===") initial_pdf_path = os.path.join(self.work_dir, "initial_draft.pdf") with open( os.path.join(self.work_dir, "initial_draft.tex"), "w", encoding="utf-8" ) as f: f.write(self.current_tex) compile_latex(self.work_dir, initial_pdf_path, "initial_draft") print(f" >> Initial PDF saved to {initial_pdf_path}") if not os.path.exists(initial_pdf_path): print(" >> Error: Initial PDF failed to compile. Cannot review.") return f" >> Error: Initial PDF '{initial_pdf_path}' failed to compile. Cannot review." v0_screenshots_dir = os.path.join( self.work_dir, "pdf_screenshots", "content_refinement", "v0" ) print(f" >> Generating snapshots for V0...") pdf_to_grid_images(initial_pdf_path, v0_screenshots_dir) current_peer_review, self.current_score = self._get_peer_review( initial_pdf_path, "initial" ) self.initial_score = self.current_score final_valid_pdf_path = initial_pdf_path # --- Content Refinement Loop --- for i in range(self.max_reflections): print(f"\n=== Refinement Iteration {i+1}/{self.max_reflections} ===") cur_filename = f"refined_paper_v{i+1}" cur_pdf_path = os.path.join(self.work_dir, f"{cur_filename}.pdf") previous_log_str = ( json.dumps(self.worklogs, indent=4) if self.worklogs else "None" ) user_prompt_text = f""" REFLECTION ITERATION {i+1}. --- PREVIOUS ITERATION LOG --- {previous_log_str} --- CURRENT REVIEWER FEEDBACK (Score: {self.current_score}) --- {json.dumps(current_peer_review, indent=2)} --- CONFERENCE GUIDELINES --- {context_files['guidelines']} --- EXPERIMENTAL LOG (Data Truth) --- {context_files['experimental_log']} --- CITATION MAP (Reference Truth) --- {context_files['citations']} --- CURRENT LATEX SOURCE --- {self.current_tex} INSTRUCTION: 1. Your goal is to IMPROVE the reviewer score (Current: {self.current_score}). 2. Address the 'Weaknesses' and 'Questions' in the Reviewer Feedback above. 3. Output the JSON Worklog first, then the Full Revised LaTeX. """ valid_pdf_for_context = final_valid_pdf_path with open(valid_pdf_for_context, "rb") as f: pdf_bytes = f.read() print(" >> calling Refinement Agent...") try: response_dict = call_gemini_with_contents( contents=[ types.Part.from_bytes( data=pdf_bytes, mime_type="application/pdf" ), types.Part.from_text(text=user_prompt_text), ], generation_configs={"system_instruction": self.system_prompt}, model_name=self.model_name, result_parsing_func=parse_gemini_json_latex_response, ) parsed = json.loads(response_dict["parsed_response"]) new_tex = parsed["latex"] if not new_tex: print(" >> Error: No LaTeX generated.") break with open( os.path.join(self.work_dir, f"{cur_filename}.tex"), "w", encoding="utf-8", ) as f: f.write(new_tex) print(" >> Compiling new draft...") compile_latex(self.work_dir, cur_pdf_path, cur_filename) if not os.path.exists(cur_pdf_path): print(" >> Compilation Failed. Retrying with previous version...") continue print(f" >> New PDF saved to {cur_pdf_path}") screenshots_dir = os.path.join( self.work_dir, "pdf_screenshots", "content_refinement", f"v{i+1}" ) print(f" >> Generating snapshots for v{i+1}...") pdf_to_grid_images(cur_pdf_path, screenshots_dir) new_review, new_score = self._get_peer_review(cur_pdf_path, f"v{i+1}") # --- COMPARISON LOGIC & METRICS EXTRACTION --- old_metrics = self._extract_scores(current_peer_review) new_metrics = self._extract_scores(new_review) excluded_axes = ["Overall", "Confidence"] comparison_axes = [ k for k in new_metrics.keys() if k not in excluded_axes ] total_gain = 0.0 total_drop = 0.0 deltas = {} for k in comparison_axes: delta = new_metrics.get(k, 0) - old_metrics.get(k, 0) deltas[k] = delta if delta > 0: total_gain += delta elif delta < 0: total_drop += abs(delta) current_log_entry = { "round": i + 1, "agent_plan": parsed["json"], "scores_before": old_metrics, "scores_after": new_metrics, "deltas": deltas, "total_gain": total_gain, "total_drop": total_drop, "outcome": "UNKNOWN", # Will be updated below } print(f" >> Score Analysis v{i+1}:") print( f" Overall: {old_metrics['Overall']} -> {new_metrics['Overall']}" ) print(f" Sub-axes Gain: +{total_gain} | Drop: -{total_drop}") print(f" Sub-axes Deltas: {deltas}") # Case 1: Overall Increased -> SUCCESS -> ACCEPT & CONTINUE if new_metrics["Overall"] > old_metrics["Overall"]: print( " >> SUCCESS: Overall Score increased! Progressing to next iteration..." ) current_log_entry["outcome"] = "ACCEPTED_SCORE_INCREASE" self.worklogs[f"v{i+1}"] = current_log_entry self.current_tex = new_tex self.current_score = new_score current_peer_review = new_review final_valid_pdf_path = cur_pdf_path # Save Log and Continue loop with open(worklog_save_path, "w", encoding="utf-8") as f: json.dump(self.worklogs, f, indent=4) # Case 2: Overall Decreased -> FAIL -> BREAK & REVERT elif new_metrics["Overall"] < old_metrics["Overall"]: print( " >> STOP: Overall Score decreased. Reverting to previous iteration." ) current_log_entry["outcome"] = "REJECTED_SCORE_DECREASE" self.worklogs[f"v{i+1}"] = current_log_entry # Save Log and Break with open(worklog_save_path, "w", encoding="utf-8") as f: json.dump(self.worklogs, f, indent=4 - methods/agents/literature_review_agent.py (22575 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import re import datetime import threading from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List, Optional, Dict from pydantic import BaseModel from google.genai import types # type: ignore from methods.prompts.literature_review_agent import ( literature_review_agent_writter_prompt, ) from utils.gemini_utils import call_gemini_with_contents, parse_gemini_latex_results from utils.scholar_utils import s2_title_search class CandidatePaper(BaseModel): title: str year: int reason: str class DiscoveryResult(BaseModel): section_name: str candidates: List[CandidatePaper] class PaperData(BaseModel): citation_key: str title: str authors: List[str] venue: str year: int abstract: str citation_count: int found_in_section: str reason: str journal: Optional[str] = None volume: Optional[str] = None pages: Optional[str] = None publication_date: Optional[str] = None class WriterOutput(BaseModel): sections_written: List[str] latex_code: str class HybridLiteratureAgent: def __init__( self, idea_path: str, experimental_log_path: str, latex_template_path: str, conference_guidelines_path: str, output_dir: str, model_name: str = "gemini-3.1-pro-preview", max_workers: int = 5, ): self.output_dir = output_dir os.makedirs(output_dir, exist_ok=True) self.model_name = model_name self.max_workers = max_workers self.context_paths = { "idea.md": idea_path, "experimental_log.md": experimental_log_path, "template.tex": latex_template_path, "conference_guidelines.md": conference_guidelines_path, } self.context_files = self._load_context_files() self.google_search_tool = types.Tool(google_search=types.GoogleSearch()) self.registry_lock = threading.Lock() self.print_lock = threading.Lock() def _safe_print(self, message: str): """Thread-safe printing to prevent garbled output.""" with self.print_lock: print(message) def run( self, outline_path: str, cutoff_date: Optional[str] = None, ): if not cutoff_date: cutoff_date = datetime.datetime.now().strftime("%Y-%m") self._safe_print( f"🚀 Starting Fast Hybrid Literature Agent (Cutoff: {cutoff_date}, Threads: {self.max_workers})..." ) with open(outline_path, "r") as f: outline = json.load(f) outline_output_path = f"{self.output_dir}/outline_v1.json" unique_papers_registry: Dict[str, PaperData] = {} processed_tasks = set() # --- PHASE 1: DISCOVERY & RETRIEVAL --- self._safe_print(f"📚 Phase 1: Analyzing Plan & Executing Search Tasks...") search_tasks = self._collect_search_tasks(outline) self._safe_print( f" -> Identified {len(search_tasks)} distinct search tasks." ) self._execute_tasks_in_parallel( search_tasks, outline, unique_papers_registry, processed_tasks, cutoff_date ) self._safe_print( f"\n✨ Final Count: {len(unique_papers_registry)} unique enriched papers retrieved." ) # --- PHASE 2: ASSET GENERATION --- final_papers_list = list(unique_papers_registry.values()) bibtex_str = self._generate_bibtex(final_papers_list) citation_map = self._generate_citation_map(final_papers_list) self._safe_print("🔄 Injecting citations back into Outline...") updated_outline = self._inject_citations_into_outline( outline, unique_papers_registry ) # --- PHASE 3: WRITING --- self._safe_print("✍️ Phase 3: Writing Introduction & Related Work...") latex_content = self._synthesize_content( updated_outline, final_papers_list, cutoff_date ) # --- PHASE 4: SAVING --- self._save_outputs( latex_content, bibtex_str, citation_map, updated_outline, outline_output_path, ) self._safe_print( f"✅ Done! Literature Review Outputs saved to {self.output_dir}/" ) def _execute_tasks_in_parallel( self, tasks, outline, registry, processed_tasks_set, cutoff_date ): """Run a batch of search tasks concurrently.""" active_tasks = [] for task in tasks: task_id = f"{task['section']}:{task['focus']}" if task_id in processed_tasks_set: continue active_tasks.append(task) processed_tasks_set.add(task_id) if not active_tasks: return self._safe_print(f" 🔎 Phase 1a: Google Candidates Discovery (MaxPool=10)...") all_candidates_with_tasks = [] with ThreadPoolExecutor(max_workers=10) as executor: # Fast for Google Search! future_to_task = { executor.submit( self._process_search_task_google, task, outline, cutoff_date ): task for task in active_tasks } for future in as_completed(future_to_task): task = future_to_task[future] try: cands = future.result() if cands: for cand_item in cands: all_candidates_with_tasks.append( {"cand": cand_item, "task": task} ) except Exception as exc: self._safe_print( f" ❌ Google Search failed for '{task['focus'][:20]}...': {exc}" ) self._safe_print( f" 📊 Phase 1b: Semantic Scholar Enrichment (Rate-limited, sequential)..." ) import time for item in all_candidates_with_tasks: # wait 1s per query to avoid SS rate limits! time.sleep(1.0) self._enrich_and_register_paper( item["cand"], item["task"], registry, cutoff_date ) def _collect_search_tasks(self, outline: Dict) -> List[Dict]: tasks = [] plan = outline.get("intro_related_work_plan", {}) # 1. Introduction Macro-Tasks intro_strat = plan.get("introduction_strategy", {}) for direction in intro_strat.get("search_directions", []): tasks.append( { "section": "Introduction", "focus": direction, "context": f"Hook: {intro_strat.get('hook_hypothesis', '')}. Gap: {intro_strat.get('problem_gap_hypothesis', '')}", "search_type": "exploration", } ) # 2. Related Work Micro-Tasks rw_strat = plan.get("related_work_strategy", {}) for sub in rw_strat.get("subsections", []): queries = sub.get("limitation_search_queries", []) if not queries: queries = [ f"{sub.get('subsection_title', '')} {sub.get('methodology_cluster', '')}" ] for q in queries: tasks.append( { "section": f"Related Work: {sub.get('subsection_title', 'General')}", "focus": q, "context": f"Mission: {sub.get('sota_investigation_mission', '')}. Hypothesis: {sub.get('limitation_hypothesis', '')}", "search_type": "exploration", } ) # 3. Targeted Tasks (Must-Haves from Section Plan) blueprint = outline.get("section_plan", []) for section in blueprint: section_title = section.get("section_title", "Unknown Section") for subsection in section.get("subsections", []): sub_title = subsection.get("subsection_title", "General") bullets = subsection.get("content_bullets", []) context_str = " ".join(bullets) hints = subsection.get("citation_hints", []) for hint in hints: tasks.append( { "section": f"{section_title} - {sub_title}", "focus": hint, "context": f"Must-have citation for section covering: {context_str}", "search_type": "targeted", } ) return tasks def _inject_citations_into_outline( self, outline: Dict, registry: Dict[str, PaperData] ) -> Dict: # Group keys by section name section_to_keys = {} for p in registry.values(): sec = p.found_in_section if sec not in section_to_keys: section_to_keys[sec] = [] section_to_keys[sec].append(p.citation_key) plan = outline.get("intro_related_work_plan", {}) # Introduction intro_strat = plan.get("introduction_strategy", {}) intro_keys = section_to_keys.get("Introduction", []) intro_strat["citation_candidates"] = list(set(intro_keys)) # Related Work rw_strat = plan.get("related_work_strategy", {}) for sub in rw_strat.get("subsections", []): key = f"Related Work: {sub.get('subsection_title', 'General')}" sub["citation_candidates"] = section_to_keys.get(key, []) # Other Sections blueprint = outline.get("section_plan", []) for section in blueprint: section_title = section.get("section_title", "Unknown Section") for sub in section.get("subsections", []): sub_title = sub.get("subsection_title", "General") key = f"{section_title} - {sub_title}" sub["citation_candidates"] = section_to_keys.get(key, []) if "citation_hints" in sub: del sub["citation_hints"] return outline def _process_search_task_google(self, task, outline, cutoff_date) -> list: if task.get("search_type") == "targeted": self._safe_print( f" 🎯 Targeted Search [{task['section']}]: '{task['focus']}'" ) else: self._safe_print( f" 🔎 Exploration [{task['section']}]: '{task['focus']}'" ) retry_count = 0 while retry_count < 3: try: candidates = self._discover_candidates(task, outline, cutoff_date) if candidates: return candidates retry_count += 1 except Exception as e: retry_count += 1 return [] def _enrich_and_register_paper(self, cand, task, registry, cutoff_date): cand_title = getattr(cand, "title", None) or cand.get("title") cand_year = getattr(cand, "year", None) or cand.get("year") cand_reason = getattr(cand, "reason", None) or cand.get("reason", "") norm_title = self._normalize_title(cand_title) with self.registry_lock: if norm_title in registry: return try: s2_data = s2_title_search(cand_title, cand_year, cutoff_date) if s2_data: if not self._is_paper_allowed(s2_data, cutoff_date): return if not s2_data.get("abstract", ""): return authors_list = [a["name"] for a in s2_data.get("authors", [])] j_info = s2_data.get("journal") or {} abstract_text = s2_data.get("abstract") or "" if len(abstract_text) > 1500: abstract_text = abstract_text[:1500] + "... [Truncated]" full_obj = PaperData( citation_key=self._generate_key( s2_data.get("authors", []), s2_data.get("year"), s2_data["title"], ), title=s2_data["title"], authors=authors_list, venue=s2_data.get("venue") or "arXiv", year=s2_data.get("year") or cand_year, abstract=abstract_text, citation_count=s2_data.get("citationCount") or 0, found_in_section=task["section"], reason=cand_reason, journal=j_info.get("name"), volume=j_info.get("volume"), pages=j_info.get("pages"), publication_date=s2_data.get("publicationDate"), ) final_norm_title = self._normalize_title(full_obj.title) with self.registry_lock: if final_norm_title not in registry: registry[final_norm_title] = full_obj self._safe_print( f" -> 📚 Added enriched paper: {full_obj.title}" ) except Exception as e: pass def _is_paper_allowed(self, s2_data: Dict, cutoff_date_str: str) -> bool: if not cutoff_date_str: return True c_parts = [int(x) for x in cutoff_date_str.split("-")] c_y = c_parts[0] c_m = c_parts[1] if len(c_parts) > 1 else 12 c_d = c_parts[2] if len(c_parts) > 2 else 1 p_y, p_m, p_d = None, None, None if s2_data.get("publicationDate"): try: p_parts = [int(x) for x in str(s2_data["publicationDate"]).split("-")] p_y = p_parts[0] p_m = p_parts[1] if len(p_parts) > 1 else None p_d = p_parts[2] if len(p_parts) > 2 else None except: pass if p_y is None and s2_data.get("year"): p_y = int(s2_data["year"]) if p_y is None: return True if p_y > c_y: return False if p_y < c_y: return True if p_m is None: return True if p_m > c_m: return False if p_m < c_m - methods/agents/outline_agent.py (2244 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.common_utils import load_md_file from utils.gemini_utils import call_gemini_with_contents import json from google.genai import types # type: ignore from methods.prompts.outline_agent import outline_agent_system_prompt class OutlineAgent: def __init__( self, model_name: str = "gemini-3.1-pro-preview", cutoff_date: str = "2024-10" ): self.model_name = model_name self.cutoff_date = cutoff_date def run( self, idea_file: str, experimental_log_file: str, latex_template_file: str, conference_guidelines_file: str, output_filepath: str = None, ): contents = [ types.Part.from_text( text=outline_agent_system_prompt.format(cutoff_date=self.cutoff_date) ), types.Part.from_text(text="'idea.md':\n" + load_md_file(idea_file)), types.Part.from_text( text="'experimental_log.md':\n" + load_md_file(experimental_log_file) ), types.Part.from_text( text="'template.tex':\n" + load_md_file(latex_template_file) ), types.Part.from_text(text=load_md_file(conference_guidelines_file)), ] response_dict = call_gemini_with_contents( contents=contents, model_name=self.model_name, ) json_result = response_dict["parsed_response"] if output_filepath is not None: with open(output_filepath, "w") as f: json.dump(json_result, f, indent=4) print(f"outline.json output written to {output_filepath}") return json_result - methods/agents/plotting_agent.py (9640 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import json import base64 import concurrent.futures from utils.common_utils import load_md_file from utils.paper_banana_utils import ( retrieve_few_shot_examples, predict_figure_content, style_figure_content, generate_figure_visuals, critique_and_revise_figure, generate_figure_caption, ) def process_single_figure( fig_plan, raw_materials_dir, model_name="gemini-3.1-pro-preview", image_model_name="gemini-3-pro-image-preview", max_critic_rounds=3, ): print(f"Processing figure: {fig_plan.get('figure_id', 'Unknown')}") title = fig_plan.get("title", "") plot_type = fig_plan.get("plot_type", "").lower() data_source_str = fig_plan.get("data_source", "").lower() objective = fig_plan.get("objective", "") aspect_ratio = fig_plan.get("aspect_ratio", "16:9") if plot_type not in ["plot", "diagram"]: task_name = "plot" if "chart" in plot_type or "plot" in plot_type else "diagram" else: task_name = plot_type # Gather raw material raw_content = "" for filename in os.listdir(raw_materials_dir): if not filename.endswith(".md"): continue match = False filename_lower = filename.lower() if "idea" in data_source_str and "idea" in filename_lower: match = True if "experiment" in data_source_str and "experiment" in filename_lower: match = True if match or filename_lower in data_source_str: filepath = os.path.join(raw_materials_dir, filename) raw_content += f"\n--- {filename} ---\n" raw_content += load_md_file(filepath) if not raw_content: for filename in ["experimental_log.md", "idea_sparse.md"]: filepath = os.path.join(raw_materials_dir, filename) if os.path.exists(filepath): raw_content += f"\n--- {filename} ---\n" raw_content += load_md_file(filepath) # 1. Retrieve few-shot examples print(f"[{fig_plan.get('figure_id', 'Unknown')}] Retrieving few-shot examples...") visual_intent = f"Title: {title}\nObjective: {objective}" examples = retrieve_few_shot_examples( task_name=task_name, raw_content=raw_content, description=visual_intent, model_name=model_name, ) # 2. Plan/Predict Content print(f"[{fig_plan.get('figure_id', 'Unknown')}] Planning figure content...") figure_desc = predict_figure_content( task_name=task_name, raw_content=raw_content, description=visual_intent, examples=examples, model_name=model_name, ) # 3. Stylist Refinement print(f"[{fig_plan.get('figure_id', 'Unknown')}] Styling figure content...") styled_figure_desc = style_figure_content( task_name=task_name, raw_content=raw_content, description=visual_intent, figure_desc=figure_desc, model_name=model_name, ) # 4. Initial Generation print( f"[{fig_plan.get('figure_id', 'Unknown')}] Generating initial visual with aspect ratio {aspect_ratio}..." ) base64_img = generate_figure_visuals( task_name=task_name, figure_description=styled_figure_desc, model_name=model_name, image_model_name=image_model_name, aspect_ratio=aspect_ratio, ) current_description = styled_figure_desc current_img = base64_img # 5. Critic Iteration critic_history = [] for round_idx in range(max_critic_rounds): print( f"[{fig_plan.get('figure_id', 'Unknown')}] Running critic round {round_idx + 1}/{max_critic_rounds}..." ) try: revised_desc, suggestions = critique_and_revise_figure( task_name=task_name, raw_content=raw_content, description=visual_intent, figure_desc=current_description, base64_image=current_img, round_idx=round_idx, model_name=model_name, ) critic_history.append( { "round": round_idx, "suggestions": suggestions, "revised_description": revised_desc, } ) if suggestions.strip() == "No changes needed.": print( f"[{fig_plan.get('figure_id', 'Unknown')}] Critic round {round_idx + 1}: No changes needed. Finishing refinement." ) break print( f"[{fig_plan.get('figure_id', 'Unknown')}] Critic round {round_idx + 1}: Changes needed. Re-generating based on suggestions..." ) current_description = revised_desc new_img = generate_figure_visuals( task_name=task_name, figure_description=current_description, model_name=model_name, image_model_name=image_model_name, aspect_ratio=aspect_ratio, ) if new_img: current_img = new_img except Exception as e: print( f"[ERROR in {fig_plan.get('figure_id', 'Unknown')} Critic Round {round_idx + 1}]: {e}" ) break print(f"[{fig_plan.get('figure_id', 'Unknown')}] Generating caption...") caption = generate_figure_caption( task_name=task_name, raw_content=raw_content, description=visual_intent, figure_desc=current_description, base64_image=current_img, model_name=model_name, ) clean_description = current_description if "Detailed Description:" in clean_description: clean_description = clean_description.replace( "Detailed Description:", "" ).strip() if "**Detailed Description:**" in clean_description: clean_description = clean_description.replace( "**Detailed Description:**", "" ).strip() print(f"[{fig_plan.get('figure_id', 'Unknown')}] Figure processing complete.") return { "figure_id": fig_plan.get("figure_id"), "title": title, "task_name": task_name, "description": clean_description, "caption": caption, "aspect_ratio": aspect_ratio, "critic_history": critic_history, "base64_img": current_img, } class PlottingAgent: def __init__( self, model_name: str = "gemini-3.1-pro-preview", image_model_name: str = "gemini-3-pro-image-preview", max_critic_rounds: int = 3, ): self.model_name = model_name self.image_model_name = image_model_name self.max_critic_rounds = max_critic_rounds def run( self, outline_json_path: str, raw_materials_dir: str, output_filepath: str = None, ): with open(outline_json_path, "r", encoding="utf-8") as f: outline_data = json.load(f) plotting_plan = outline_data.get("plotting_plan", []) results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: futures = { executor.submit( process_single_figure, plan, raw_materials_dir, self.model_name, self.image_model_name, self.max_critic_rounds, ): plan for plan in plotting_plan } for future in concurrent.futures.as_completed(futures): try: res = future.result() results.append(res) except Exception as e: plan = futures[future] import traceback print(f"[ERROR] Failed processing {plan.get('figure_id')}: {e}") traceback.print_exc() if output_filepath: output_dir = os.path.dirname(output_filepath) os.makedirs(output_dir, exist_ok=True) figures_dir = os.path.join(output_dir, "figures") os.makedirs(figures_dir, exist_ok=True) for res in results: if res.get("base64_img"): fig_id_clean = res["figure_id"].replace(" ", "_").lower() img_path = os.path.join(figures_dir, f"{fig_id_clean}.jpg") try: with open(img_path, "wb") as fh: fh.write(base64.b64decode(res["base64_img"])) print(f"Saved {img_path}") res["image_path"] = f"figures/{fig_id_clean}.jpg" except Exception as e: print(f"Failed to save image {fig_id_clean}: {e}") if "base64_img" in res: del res["base64_img"] with open(output_filepath, "w", encoding="utf-8") as f: json.dump(results, f, indent=4) return results - methods/agents/section_writing_agent.py (8190 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This agent takes in a latex file with partially filled sections and fills in the missing sections. Input: - outline.json: A JSON file containing the outline of the paper, including section headings and key points to cover in each section. - template.tex: A LaTeX file with some sections (intro & related work) filled and others missing. - idea.md: A detailed summary of the methodology, core contributions, and theoretical framework. - experimental_log.md: A summary of experimental results, including raw data points, ablation studies, and performance metrics. - conference_guidelines.md: The submission guidelines for the target conference. - figures: A list of figures to be included in the paper, each with: - figure_path: str - figure_caption: str Output: - completed_paper.tex: The completed LaTeX file with all sections filled in. """ import json from typing import List, Dict from methods.prompts.section_writing_agent import section_writing_agent_prompt from utils.gemini_utils import parse_gemini_latex_results from utils.llm_backend_utils import call_llm_with_images import json from typing import List, Dict class SectionWritingAgent: def __init__( self, outline_path: str, template_path: str, idea_path: str, experimental_log_path: str, citation_map_path: str, figures_info_path: str, guidelines_path: str, model_name: str = "gemini-3.1-pro-preview", ): self.model_name = model_name self.system_prompt = section_writing_agent_prompt self.outline_path = outline_path self.template_path = template_path self.idea_path = idea_path self.experimental_log_path = experimental_log_path self.citation_map_path = citation_map_path self.figures_info_path = figures_info_path self.guidelines_path = guidelines_path def _read_file(self, file_path: str) -> str: try: with open(file_path, "r", encoding="utf-8") as f: return f.read() except FileNotFoundError: return f"[Error: File {file_path} not found]" def _read_json(self, file_path: str) -> Dict: try: with open(file_path, "r", encoding="utf-8") as f: return json.load(f) except (FileNotFoundError, json.JSONDecodeError): return {} def _format_figures_context(self, figures_info: List[Dict]) -> str: """ Parses the specific figures_info.json format. Extracts 'name' -> used as filename (now includes extension). Extracts 'caption' -> used as caption. """ if not figures_info: return "No figures provided." context = "### AVAILABLE FIGURES LIST\n" for idx, fig in enumerate(figures_info, 1): name = fig.get("name", f"figure_{idx}.jpg") if ( name and not name.endswith(".png") and not name.endswith(".jpg") and not name.endswith(".pdf") ): name += ".png" caption = fig.get("caption", "No caption provided.") context += f"Item {idx}:\n" context += f" - Filename: {name}\n" context += f" - Caption: {caption}\n" context += "--------------------------------------------------\n" return context def _format_citation_library(self, citation_map: Dict) -> str: if not citation_map: return "No citation data provided." context = "### REFERENCE LIBRARY (Use these keys for \\cite{})\n" for key, data in citation_map.items(): title = data.get("title", "Unknown Title") authors_list = data.get("authors", []) authors = ", ".join(authors_list[:3]) if len(authors_list) > 3: authors += " et al." year = data.get("year", "N/A") abstract = data.get("abstract", "No abstract available.") context += f"--- Key: {key} ---\n" context += f"Title: {title}\n" context += f"Authors: {authors} ({year})\n" context += f"Abstract: {abstract}\n\n" return context def run(self, output_path: str) -> str: print(f" > Agent {self.__class__.__name__} starting...") try: outline_content = self._read_file(self.outline_path) template_content = self._read_file(self.template_path) idea_content = self._read_file(self.idea_path) log_content = self._read_file(self.experimental_log_path) guidelines_content = self._read_file(self.guidelines_path) citation_map = self._read_json(self.citation_map_path) citation_library_content = self._format_citation_library(citation_map) figures_info = self._read_json(self.figures_info_path) figures_content = self._format_figures_context(figures_info) import os figures_dir = os.path.dirname(self.figures_info_path) image_paths = [] if figures_info: for fig in figures_info: name = fig.get("name", "") if name: if ( not name.endswith(".png") and not name.endswith(".jpg") and not name.endswith(".pdf") ): name += ".png" img_path = os.path.join(figures_dir, name) if os.path.exists(img_path) and not name.endswith( ".pdf" ): # We can only pass raster images, not PDFs, to Vertex Vision image_paths.append(img_path) except Exception as e: return f"Error reading input files: {str(e)}" user_prompt = f""" Here are the input materials for the paper. --- INPUT: outline.json (Structure & Citation Candidates) --- {outline_content} --- INPUT: citation_map.json (Reference Library) --- {citation_library_content} --- INPUT: idea.md (Technical Method Details) --- {idea_content} --- INPUT: experimental_log.md (Raw Data for Tables) --- {log_content} --- INPUT: figures_list (Available Figure Files) --- {figures_content} --- INPUT: conference_guidelines.md --- {guidelines_content} --- INPUT: template.tex (DO NOT CHANGE EXISTING TEXT. FILL MISSING SECTIONS.) --- {template_content} **INSTRUCTION**: 1. Identify missing sections in `template.tex`. 2. Generate LaTeX content for those sections based on `outline.json`. 3. Construct tables using data from `experimental_log.md`. 4. Insert figures using the filenames from the 'figures_list' input. 5. Use citations from the Reference Library. 6. Return the **full** compilable LaTeX file. """ print(" > Generating paper content with Multimodal Images...") response_dict = call_llm_with_images( prompt=user_prompt, images=image_paths, generation_configs={ "system_instruction": self.system_prompt, "temperature": 0.7, }, model_name=self.model_name, result_parsing_func=parse_gemini_latex_results, check_parsed_response_not_none=True, ) cleaned_latex = response_dict["parsed_response"] with open(output_path, "w", encoding="utf-8") as f: f.write(cleaned_latex) print(f" > Paper saved to {output_path}") return cleaned_latex - methods/paper_writer_with_plotting.py (20016 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path as osp import shutil import traceback import json import time import tqdm from concurrent.futures import ProcessPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor from methods.agents.outline_agent import OutlineAgent from methods.agents.literature_review_agent import HybridLiteratureAgent from methods.agents.section_writing_agent import SectionWritingAgent from methods.agents.content_refinement_agent import ContentRefinementAgent from methods.agents.plotting_agent import PlottingAgent from utils.common_utils import ( create_log_folder, ) def write_single_paper( base_dir, latex_template_dir, idea_filename="idea_dense.md", experimental_log_filename="experimental_log.md", writer_model_name: str = "gemini-3.1-pro-preview", reflection_model_name: str = "gemini-3.1-pro-preview", plotting_model_name: str = "gemini-3.1-pro-preview", image_model_name: str = "gemini-3-pro-image-preview", research_cutoff: str = "2024-11", plotting_max_critic_rounds: int = 3, ): latex_writeup_dir = osp.join(base_dir, "latex_writeup") pdf_file = osp.join(base_dir, "final_paper.pdf") time_log = {} time_log["Start_Time_Absolute"] = time.time() if osp.exists(latex_writeup_dir): shutil.rmtree(latex_writeup_dir) if osp.exists(pdf_file): os.remove(pdf_file) materials_dir = osp.join(base_dir, "raw_materials") figures_dir = osp.join(latex_writeup_dir, "figures") os.makedirs(figures_dir, exist_ok=True) max_n_tries = 3 cur_try = 0 succeed = False while (not succeed) and (cur_try < max_n_tries): try: assert osp.exists( materials_dir ), f"Please make sure '{materials_dir}' exists!" shutil.copytree(latex_template_dir, latex_writeup_dir, dirs_exist_ok=True) guidelines_path = osp.join(latex_writeup_dir, "guidelines.md") latex_template_path = osp.join(latex_writeup_dir, "template.tex") idea_file_path = osp.join(materials_dir, idea_filename) experimental_log_file_path = osp.join( materials_dir, experimental_log_filename ) assert osp.exists( guidelines_path ), f"No guidelines file detected at '{guidelines_path}'!" assert osp.exists( latex_template_path ), f"No template file detected at '{latex_template_path}'!" assert osp.exists( idea_file_path ), f"No idea file detected at '{idea_file_path}'!" assert osp.exists( experimental_log_file_path ), f"No experimental log file detected at '{experimental_log_file_path}'!" ############################################################################################### # Step 1: Call outline agent _t0 = time.time() outline_agent = OutlineAgent( model_name=writer_model_name, cutoff_date=research_cutoff ) outline_file_path = osp.join(base_dir, "outline.json") outline_agent.run( idea_file=idea_file_path, experimental_log_file=experimental_log_file_path, latex_template_file=latex_template_path, conference_guidelines_file=guidelines_path, output_filepath=outline_file_path, ) time_log["OutlineAgent"] = time.time() - _t0 ############################################################################################### # Step 2: Call HybridLiteratureAgent and PlottingAgent in parallel literature_agent_output_dir = osp.join(base_dir, "literature_agent_output") def run_literature_agent(): print("[DEBUG] Starting run_literature_agent") literature_agent = HybridLiteratureAgent( idea_path=idea_file_path, experimental_log_path=experimental_log_file_path, latex_template_path=latex_template_path, conference_guidelines_path=guidelines_path, output_dir=literature_agent_output_dir, model_name=writer_model_name, max_workers=3, ) literature_agent.run( outline_path=outline_file_path, cutoff_date=research_cutoff, ) print("[DEBUG] Finished run_literature_agent") return True def run_plotting_agent(): print("[DEBUG] Starting run_plotting_agent") plotting_results_path = osp.join(base_dir, "plotting_results.json") plotting_agent = PlottingAgent( model_name=plotting_model_name, image_model_name=image_model_name, max_critic_rounds=plotting_max_critic_rounds, ) res = plotting_agent.run( outline_json_path=outline_file_path, raw_materials_dir=materials_dir, output_filepath=plotting_results_path, ) print("[DEBUG] Finished run_plotting_agent") return res _t1 = time.time() with ThreadPoolExecutor(max_workers=2) as executor: future_lit = executor.submit(run_literature_agent) future_plot = executor.submit(run_plotting_agent) future_lit.result() plot_results = future_plot.result() time_log["Parallel_Lit_and_Plot"] = time.time() - _t1 figures_info = [] for plot_obj in plot_results: if "image_path" in plot_obj: src_plot = osp.join(base_dir, plot_obj["image_path"]) dst_plot = osp.join(latex_writeup_dir, plot_obj["image_path"]) if osp.exists(src_plot): os.makedirs(osp.dirname(dst_plot), exist_ok=True) shutil.copy(src_plot, dst_plot) clean_name = osp.basename(dst_plot) figures_info.append( {"name": clean_name, "caption": plot_obj.get("caption", "")} ) with open(osp.join(figures_dir, "info.json"), "w") as f: json.dump(figures_info, f, indent=4) # Copy literature agent outputs shutil.copy( osp.join(literature_agent_output_dir, "outline_v1.json"), outline_file_path, ) shutil.copy( osp.join(literature_agent_output_dir, "updated_template.tex"), osp.join(latex_writeup_dir, "template.tex"), ) shutil.copy( osp.join(literature_agent_output_dir, "references.bib"), osp.join(latex_writeup_dir, "references.bib"), ) ############################################################################################### # Step 3. Call SectionWritingAgent citation_map_path = osp.join( literature_agent_output_dir, "citation_map.json" ) _t2 = time.time() section_writing_agent = SectionWritingAgent( outline_path=outline_file_path, template_path=osp.join(latex_writeup_dir, "template.tex"), idea_path=idea_file_path, experimental_log_path=experimental_log_file_path, citation_map_path=citation_map_path, figures_info_path=osp.join(figures_dir, "info.json"), guidelines_path=guidelines_path, model_name=writer_model_name, ) raw_draft_tex_path = osp.join(latex_writeup_dir, "raw_draft_paper.tex") section_writing_agent.run(output_path=raw_draft_tex_path) time_log["SectionWritingAgent"] = time.time() - _t2 ############################################################################################### # Step 4: Content Reflection content_refinement_workdir = os.path.join( base_dir, "content_refinement_workdir" ) if os.path.exists(content_refinement_workdir): shutil.rmtree(content_refinement_workdir) os.makedirs(content_refinement_workdir, exist_ok=True) shutil.copytree( latex_writeup_dir, content_refinement_workdir, dirs_exist_ok=True, ) _t3 = time.time() content_refinement_agent = ContentRefinementAgent( experimental_log_path=experimental_log_file_path, citation_map_path=citation_map_path, guidelines_path=guidelines_path, model_name=reflection_model_name, max_reflections=3, work_dir=content_refinement_workdir, ) content_refinment_agent_output_pdf_path = content_refinement_agent.run( texfile_path=raw_draft_tex_path ) time_log["ContentRefinementAgent"] = time.time() - _t3 time_log["Total_Time_Span"] = time.time() - time_log["Start_Time_Absolute"] with open(osp.join(base_dir, "time_breakdown.json"), "w") as f: json.dump(time_log, f, indent=4) assert osp.exists( content_refinment_agent_output_pdf_path ), f"Content Refinement Agent failed to produce final PDF at {content_refinment_agent_output_pdf_path}!" final_pdf_path = osp.join(base_dir, "final_paper.pdf") print( f" >> Copying final PDF from {content_refinment_agent_output_pdf_path} to {final_pdf_path}...\n" ) shutil.copy(content_refinment_agent_output_pdf_path, final_pdf_path) print(" >> Final PDF saved to:\n", final_pdf_path) succeed = True return True except Exception: print(f"[Try {cur_try+1}/{max_n_tries}] EXCEPTION in perform_writeup:") print(traceback.format_exc()) cur_try += 1 if cur_try < max_n_tries: print("Try again now...") print(f"Writeup failed after {max_n_tries}! Stop here.") return False def process_paper_task( paper, output_dir, latex_template_dir, idea_filename, experimental_log_filename, writer_model_name, plotting_model_name, image_model_name, plotting_max_critic_rounds, research_cutoff, ): """ Worker function to process a single paper. Returns a dictionary with results to be handled by the main thread. """ try: paper_id = paper["paper_id"] print(f"Starting processing for paper '{paper_id}'...") paper_dest_dir = osp.join(output_dir, paper_id) os.makedirs(paper_dest_dir, exist_ok=True) final_paper_path = osp.join(paper_dest_dir, "final_paper.pdf") result = { "paper_id": paper_id, "status": "failed", "paper_path": "", } if True: shutil.rmtree(paper_dest_dir, ignore_errors=True) os.makedirs(paper_dest_dir, exist_ok=True) content_dest_dir = osp.join(paper_dest_dir, "raw_materials") os.makedirs(content_dest_dir, exist_ok=True) print( f"Copying content_folder from {paper['content_folder']} to {content_dest_dir}..." ) shutil.copytree( src=paper["content_folder"], dst=content_dest_dir, dirs_exist_ok=True ) print(f"Copying original paper to {content_dest_dir}...") shutil.copy( src=paper["pdf_path"], dst=osp.join(content_dest_dir, "original_paper.pdf"), ) success = write_single_paper( base_dir=paper_dest_dir, latex_template_dir=latex_template_dir, idea_filename=idea_filename, experimental_log_filename=experimental_log_filename, writer_model_name=writer_model_name, plotting_model_name=plotting_model_name, image_model_name=image_model_name, plotting_max_critic_rounds=plotting_max_critic_rounds, research_cutoff=research_cutoff, ) if success: result["status"] = "success" result["paper_path"] = final_paper_path print(f"Paper {paper_id} successfully written!") else: print(f"Paper {paper_id} writing failed.") return result return result except Exception as e: print(f"CRITICAL ERROR in worker for paper {paper['paper_id']}: {e}") print(traceback.format_exc()) return {"paper_id": paper["paper_id"], "status": "error", "error": str(e)} def _load_papers_from_config( process_root_folder: bool, root_input_folder: str, paper_info_path: str, n_papers: int, ): """Loads a list of paper dictionaries from either a root folder or a JSON file.""" if process_root_folder: if not root_input_folder: raise ValueError( "Please provide root_input_folder if process_root_folder is True." ) all_papers_candidates = [] all_paper_ids_raw = os.listdir(root_input_folder) all_paper_ids = [ paper_id for paper_id in all_paper_ids_raw if os.path.isdir(os.path.join(root_input_folder, paper_id)) ] for paper_id in all_paper_ids: content_folder = os.path.join(root_input_folder, paper_id, "raw_materials") pdf_path = os.path.join(root_input_folder, paper_id, f"{paper_id}.pdf") if not os.path.exists(content_folder) or not os.path.exists(pdf_path): continue paper_entry = { "paper_id": paper_id, "content_folder": content_folder, "pdf_path": pdf_path, } all_papers_candidates.append(paper_entry) else: - methods/paper_writer.py (16572 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path as osp import shutil import traceback import json import tqdm from concurrent.futures import ProcessPoolExecutor, as_completed from methods.agents.outline_agent import OutlineAgent from methods.agents.literature_review_agent import HybridLiteratureAgent from methods.agents.section_writing_agent import SectionWritingAgent from methods.agents.content_refinement_agent import ContentRefinementAgent from utils.common_utils import ( create_log_folder, ) def write_single_paper( base_dir, latex_template_dir, idea_filename="idea_dense.md", experimental_log_filename="experimental_log.md", writer_model_name: str = "gemini-3.1-pro-preview", reflection_model_name: str = "gemini-3.1-pro-preview", research_cutoff: str = "2024-11", ): latex_writeup_dir = osp.join(base_dir, "latex_writeup") pdf_file = osp.join(base_dir, "final_paper.pdf") if osp.exists(latex_writeup_dir): shutil.rmtree(latex_writeup_dir) if osp.exists(pdf_file): os.remove(pdf_file) materials_dir = osp.join(base_dir, "raw_materials") old_figure_dir = osp.join(materials_dir, "figures") new_figure_dir = osp.join(latex_writeup_dir, "figures") if osp.exists(old_figure_dir): shutil.copytree(old_figure_dir, new_figure_dir, dirs_exist_ok=True) figures_dir = new_figure_dir max_n_tries = 3 cur_try = 0 succeed = False while (not succeed) and (cur_try < max_n_tries): try: assert osp.exists( materials_dir ), f"Please make sure '{materials_dir}' exists!" shutil.copytree(latex_template_dir, latex_writeup_dir, dirs_exist_ok=True) guidelines_path = osp.join(latex_writeup_dir, "guidelines.md") latex_template_path = osp.join(latex_writeup_dir, "template.tex") idea_file_path = osp.join(materials_dir, idea_filename) experimental_log_file_path = osp.join( materials_dir, experimental_log_filename ) assert osp.exists( guidelines_path ), f"No guidelines file detected at '{guidelines_path}'!" assert osp.exists( latex_template_path ), f"No template file detected at '{latex_template_path}'!" assert osp.exists( idea_file_path ), f"No idea file detected at '{idea_file_path}'!" assert osp.exists( experimental_log_file_path ), f"No experimental log file detected at '{experimental_log_file_path}'!" ############################################################################################### # Step 1: Call outline agent outline_agent = OutlineAgent( model_name=writer_model_name, cutoff_date=research_cutoff ) outline_file_path = osp.join(base_dir, "outline.json") outline_agent.run( idea_file=idea_file_path, experimental_log_file=experimental_log_file_path, latex_template_file=latex_template_path, conference_guidelines_file=guidelines_path, output_filepath=outline_file_path, ) ############################################################################################### # Step 2: Call HybridLiteratureAgent literature_agent_output_dir = osp.join(base_dir, "literature_agent_output") literature_agent = HybridLiteratureAgent( idea_path=idea_file_path, experimental_log_path=experimental_log_file_path, latex_template_path=latex_template_path, conference_guidelines_path=guidelines_path, output_dir=literature_agent_output_dir, model_name=writer_model_name, max_workers=3, ) literature_agent.run( outline_path=outline_file_path, cutoff_date=research_cutoff, ) shutil.copy( osp.join(literature_agent_output_dir, "outline_v1.json"), outline_file_path, ) shutil.copy( osp.join(literature_agent_output_dir, "updated_template.tex"), osp.join(latex_writeup_dir, "template.tex"), ) shutil.copy( osp.join(literature_agent_output_dir, "references.bib"), osp.join(latex_writeup_dir, "references.bib"), ) ############################################################################################### # Step 3. Call SectionWritingAgent citation_map_path = osp.join( literature_agent_output_dir, "citation_map.json" ) section_writing_agent = SectionWritingAgent( outline_path=outline_file_path, template_path=osp.join(latex_writeup_dir, "template.tex"), idea_path=idea_file_path, experimental_log_path=experimental_log_file_path, citation_map_path=citation_map_path, figures_info_path=osp.join(figures_dir, "info.json"), guidelines_path=guidelines_path, model_name=writer_model_name, ) raw_draft_tex_path = osp.join(latex_writeup_dir, "raw_draft_paper.tex") section_writing_agent.run(output_path=raw_draft_tex_path) ############################################################################################### # Step 4: Content Reflection content_refinement_workdir = os.path.join( base_dir, "content_refinement_workdir" ) if os.path.exists(content_refinement_workdir): shutil.rmtree(content_refinement_workdir) os.makedirs(content_refinement_workdir, exist_ok=True) shutil.copytree( latex_writeup_dir, content_refinement_workdir, dirs_exist_ok=True, ) content_refinement_agent = ContentRefinementAgent( experimental_log_path=experimental_log_file_path, citation_map_path=citation_map_path, guidelines_path=guidelines_path, model_name=reflection_model_name, max_reflections=3, work_dir=content_refinement_workdir, ) content_refinment_agent_output_pdf_path = content_refinement_agent.run( texfile_path=raw_draft_tex_path ) assert osp.exists( content_refinment_agent_output_pdf_path ), f"Content Refinement Agent failed to produce final PDF at {content_refinment_agent_output_pdf_path}!" final_pdf_path = osp.join(base_dir, "final_paper.pdf") print( f" >> Copying final PDF from {content_refinment_agent_output_pdf_path} to {final_pdf_path}...\n" ) shutil.copy(content_refinment_agent_output_pdf_path, final_pdf_path) print(" >> Final PDF saved to:\n", final_pdf_path) succeed = True return True except Exception: print(f"[Try {cur_try+1}/{max_n_tries}] EXCEPTION in perform_writeup:") print(traceback.format_exc()) cur_try += 1 if cur_try < max_n_tries: print("Try again now...") print(f"Writeup failed after {max_n_tries}! Stop here.") return False def process_paper_task( paper, output_dir, latex_template_dir, idea_filename, experimental_log_filename, writer_model_name, research_cutoff, ): """ Worker function to process a single paper. Returns a dictionary with results to be handled by the main thread. """ try: paper_id = paper["paper_id"] print(f"Starting processing for paper '{paper_id}'...") paper_dest_dir = osp.join(output_dir, paper_id) os.makedirs(paper_dest_dir, exist_ok=True) final_paper_path = osp.join(paper_dest_dir, "final_paper.pdf") result = { "paper_id": paper_id, "status": "failed", "paper_path": "", } if True: shutil.rmtree(paper_dest_dir, ignore_errors=True) os.makedirs(paper_dest_dir, exist_ok=True) content_dest_dir = osp.join(paper_dest_dir, "raw_materials") os.makedirs(content_dest_dir, exist_ok=True) print( f"Copying content_folder from {paper['content_folder']} to {content_dest_dir}..." ) shutil.copytree( src=paper["content_folder"], dst=content_dest_dir, dirs_exist_ok=True ) print(f"Copying original paper to {content_dest_dir}...") shutil.copy( src=paper["pdf_path"], dst=osp.join(content_dest_dir, "original_paper.pdf"), ) success = write_single_paper( base_dir=paper_dest_dir, latex_template_dir=latex_template_dir, idea_filename=idea_filename, experimental_log_filename=experimental_log_filename, writer_model_name=writer_model_name, research_cutoff=research_cutoff, ) if success: result["status"] = "success" result["paper_path"] = final_paper_path print(f"Paper {paper_id} successfully written!") else: print(f"Paper {paper_id} writing failed.") return result return result except Exception as e: print(f"CRITICAL ERROR in worker for paper {paper['paper_id']}: {e}") print(traceback.format_exc()) return {"paper_id": paper["paper_id"], "status": "error", "error": str(e)} def _load_papers_from_config( process_root_folder: bool, root_input_folder: str, paper_info_path: str, n_papers: int, ): """Loads a list of paper dictionaries from either a root folder or a JSON file.""" if process_root_folder: if not root_input_folder: raise ValueError( "Please provide root_input_folder if process_root_folder is True." ) all_papers_candidates = [] all_paper_ids_raw = os.listdir(root_input_folder) all_paper_ids = [ paper_id for paper_id in all_paper_ids_raw if os.path.isdir(os.path.join(root_input_folder, paper_id)) ] for paper_id in all_paper_ids: content_folder = os.path.join(root_input_folder, paper_id, "raw_materials") pdf_path = os.path.join(root_input_folder, paper_id, f"{paper_id}.pdf") if not os.path.exists(content_folder) or not os.path.exists(pdf_path): continue paper_entry = { "paper_id": paper_id, "content_folder": content_folder, "pdf_path": pdf_path, } all_papers_candidates.append(paper_entry) else: if not paper_info_path: raise ValueError( "Please provide paper_info_path if process_root_folder is False." ) with open(paper_info_path, "r") as f: all_papers_candidates = json.load(f) if n_papers < 0: n_papers = len(all_papers_candidates) else: n_papers = min(n_papers, len(all_papers_candidates)) return all_papers_candidates[:n_papers] def _load_logs(output_dir: str): """Loads existing logs to support resuming a batch job.""" paper_writing_log_path = osp.join(output_dir, "paper_writing_log.json") write_log = {} if osp.exists(paper_writing_log_path): with open(paper_writing_log_path, "r") as f: write_log = json.load(f) print(f"Loaded existing writing log with {len(write_log)} entries.") return write_log def _filter_and_prepare_papers( all_papers, write_log, process_root_folder, paper_info_path ): """Filters out completed papers and resolves paths.""" papers_to_process = [] base_info_dir = "" if (not process_root_folder) and (paper_info_path != ""): base_info_dir = osp.dirname(paper_info_path) for paper in all_papers: # Check if already successfully processed in log if ( paper["paper_id"] in write_log and write_log[paper["paper_id"]].get("status") == "success" and osp.exists(write_log[paper["paper_id"]].get("paper_path", "")) ): continue if not osp.isabs(paper["content_folder"]): paper["content_folder"] = osp.join(base_info_dir, paper["content_folder"]) if not osp.isabs(paper["pdf_path"]): paper["pdf_path"] = osp.join(base_info_dir, paper["pdf_path"]) papers_to_process.append(paper) return papers_to_process def run_batch_writeup( latex_template_dir: str, process_root_folder: bool = True, root_input_folder: str = "", paper_info_path: str = "", output_dir: str = "", output_dir_prefix: str = "", date_str: str = "", idea_filename: str = "idea_sparse.md", experimental_log_filename: str = "experimental_log.md", writer_model_name: str = "gemini-3.1-pro-preview", research_cutoff: str = "2024-11", n_papers=-1, max_workers=4, ): if output_dir == "": output_dir = create_log_folder(prefix=output_dir_prefix, date_str=date_str) else: print(f"Resuming from output_dir = {output_dir}") os.makedirs(output_dir, exist_ok=True) all_papers = _load_papers_from_config( process_root_folder, root_input_folder, paper_info_path, n_papers ) write_log = _load_logs(output_dir) papers_to_process = _filter_and_prepare_papers( all_papers, write_log, process_root_folder, paper_info_path ) print( f"Starting batch writeup for {len(papers_to_process)} papers (skipped {len(all_papers) - len(papers_to_process)} already done)." ) print(f"Parallel execution with max_workers={max_workers}") paper_writing_log_path = osp.join(output_dir, "paper_writing_log.json") with ProcessPoolExecutor(max_workers=max_workers) - methods/prompts/content_refinement_agent.py (4845 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.prompt_utils import UNIVERSAL_NO_LEAKAGE_PROMPT content_refinement_agent_system_prompt = """Role: Senior AI Researcher. Task: Revise and strengthen a LaTeX research paper by systematically addressing peer review feedback. You are the author responsible for the "Rebuttal via Revision" phase. You will receive: * 'paper.tex': The current LaTeX source code. * 'paper.pdf': The compiled PDF context. * 'conference_guidelines.md': The formatting and page limit rules. * 'experimental_log.md': The Ground Truth for all data and metrics. * 'worklog.json': History of previous changes. * 'citation_map.json': The allowed bibliography. * 'reviewer_feedback': A JSON object containing specific Strengths, Weaknesses, Questions, and Decisions from an LLM reviewer. YOUR GOAL: 1. Analyze Feedback: Deconstruct the `reviewer_feedback` into actionable editing tasks. 2. Address Weaknesses: Rewrite sections to clarify logic, strengthen arguments, or justify design choices pointed out as weak. 3. Integrate Answers: Incorporate answers to the reviewer's "Questions" directly into the manuscript (e.g., adding training cost details to the Implementation section). 4. Execution: Generate a JSON worklog of your editorial decisions and the full, revised LaTeX source. ### CRITICAL EXECUTION STANDARDS #### 1. Content Revision Strategy - Weakness Mitigation: If the reviewer flags "incremental novelty," rewrite the Introduction and Related Work to explicitly contrast your contribution against prior art. If they flag "unclear methodology," restructure the relevant section for clarity. - Answering Questions: Do NOT write a separate response letter. If the reviewer asks "What is the inference latency?", you must find a natural place in the paper (e.g., Experiments or Discussion) to insert that information, ensuring it aligns with `experimental_log.md`. - Preserve Strengths: Do not delete or heavily alter sections listed under "Strengths" unless necessary for space or flow. #### 2. Data Integrity & Hallucination Check - Ground Truth: All numerical claims (accuracy, parameter count, training hours, latency) MUST be verified against `experimental_log.md`. - Missing Data: If the reviewer asks for new experiments, ablations, or baselines that are NOT in `experimental_log.md`, simply ignore those specific requests. Your job is purely presentation refinement of the existing completed experiments, not adding or promising to add new experiments. #### 3. Writing Style & Tone - Academic Tone: Maintain a formal, objective, and precise tone. Avoid defensive language. - Conciseness: If the paper is near the page limit, prioritize density of information over flowery prose. - Flow: Ensure that new insertions (answers to questions) transition smoothly with existing text. #### 4. LaTeX & Citation Integrity - Structure: Do not break the LaTeX compilation. Keep packages and environments stable. If using `figure*` for wide figures, ensure they are closed with `\\end{figure*}` (not `\\end{figure}`). Check for completeness. - Citations: Use ONLY keys from `citation_map.json`. ### OUTPUT FORMAT (STRICT) You MUST return your response in two distinct code blocks in this exact order: 1. Worklog for the current turn (JSON): ```json {{ "addressed_weaknesses": [ "Clarified contribution novelty in Intro (Reviewer point 2)", "Added justification for two-stage training (Reviewer point 1)" ], "integrated_answers": [ "Added training cost (45 GPU hours) to Implementation Details", "Added epsilon hyperparameter explanation to Method section" ], "actions_taken": [ "Rewrote Section 3.2 for clarity", "Inserted new paragraph in Section 5.1 regarding latency" ] }} ``` 2.The FULL revised LaTeX code: ```latex ... Full revised LaTeX code here ... ``` ### IMPORTANT NOTES - Completeness: Always provide the FULL LaTeX code. Do not return diffs or partial snippets. - Responsiveness: Every question in the reviewer_feedback must be addressed by improving the presentation, EXCEPT for questions asking for new experiments or data not in `experimental_log.md` (which should be ignored). Never explicitly state a limitation. - Safety: Do not remove the \\documentclass or essential preamble. """ + "\n" + UNIVERSAL_NO_LEAKAGE_PROMPT - methods/prompts/format_agent.py (5010 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.prompt_utils import UNIVERSAL_NO_LEAKAGE_PROMPT format_agent_system_prompt = """Role: Senior AI Researcher. Task: Polish and debug a LaTeX research paper by comparing the rendered PDF against conference guidelines. You are the final gatekeeper. You will receive: * 'paper.tex': The current LaTeX source code of the research paper. * 'paper.pdf': The current compiled PDF of the research paper. * 'conference_guidelines.md': The official guidelines of the target conference. * 'experimental_log.md': The experimental log containing raw data. * 'worklog.json': A JSON worklog of previous iterations (if any). * 'citation_map.json': A Reference Library containing BibTeX keys, titles, and abstracts of cited papers. All of your citations must match these keys. Do NOT add new citations that are not in the citation_map.json. YOUR GOAL: 1. Visual Analysis: Inspect the PDF for layout defects (overflows, overlaps, illegible text) and fix them in LaTeX. 2. Strict Enforcement: Adhere strictly to page limits, margins, and the given formatting rules. Condense the paper if it exceeds page limits. 3. Content Polishing: Correct typos and inconsistencies without altering scientific meaning. 4. Execution: Generate a JSON worklog and the full, compilable, corrected LaTeX source. ### CRITICAL EXECUTION STANDARDS #### 1. Figure Layout - Placement(CRITICAL): Ensure figures appear near or before their first reference. Make sure no figure appears after the Conclusion section. - Missing Figures: Ensure ALL figures from `figures_list` are implemented. DO NOT merge or combine multiple figures into one. - Optimization: Always prefer single-column (`figure`) to save space. - Sizing: Use `width=1.0\\linewidth` (relative to column). Never set fixed height AND width (preserve aspect ratio). #### 2. Table Readability - Placement(CRITICAL): Ensure tables appear near or before their first reference. Make sure no table appears after the Conclusion section. - Overflow/Overlap: If cell content overlaps or exceeds margins: - Apply `\\small` or `\\footnotesize`. - Abbreviate headers. - Switch to `tabularx` with `X` columns for auto-wrapping. - Data Check: If table values differ from `experimental_log.md`, correct them. #### 3. Page Limit Enforcement (CRITICAL) - Check Length: Verify that the main content page count (counting all pages up to and including the start of the References section) complies with `conference_guidelines.md`. - If Over Limit: You MUST condense the text. - *Strategy 1*: Remove "flowery" or non-informational adjectives (e.g., "meticulously", "comprehensive"). - *Strategy 2*: Merge very short paragraphs to save vertical whitespace. - *Strategy 3*: Tighten vertical spacing (e.g., `\\vspace{{-5pt}}`) around figures/tables (use caution). - CRITICAL: Do NOT remove core contributions, main results, Abstract, or Conclusion. #### 4. Consistency & Typography - Typos: Correct spelling errors. - Citations: Ensure all `\\cite{{}}` keys exactly match the keys provided in `citation_map.json`. If a citation key doesn't match any key in `citation_map.json`, correct it or remove it. - Orphans/Widows: If a section header appears at the very bottom of a column with no text under it, insert `\\clearpage` or adjust spacing. ### OUTPUT FORMAT (STRICT) You MUST return your response in two distinct code blocks in this exact order: 1. Worklog for the current turn (JSON): ```json {{ "critical_errors": ["Figure 1 cuts into text", "Page count 9/8"], "minor_issues": ["Table 2 header alignment", "Typos xxx in Intro"], "actions_taken": [ "Changed Figure 1 to figure* environment", "Removed 3 adjectives from Abstract to save space", "Added resizebox to Table 2" ] }} ``` 2. The FULL corrected LaTeX code: ```latex ... Full correct LaTeX code here ... ``` ### IMPORTANT NOTES - Always provide the FULL LaTeX code, even if only minor changes were made. - DO NOT change '\\usepackage[capitalize]{{cleveref}}' into '\\usepackage[capitalize]{{cleverref}}', as there's no 'cleverref.sty'. - Ensure the LaTeX code compiles without errors, e.g. all the begin and end statements match correctly. - Ensure the new LaTeX code matches the indentation and spacing style of the given latex. Don't add new packages unless absolutely necessary. - Make sure there's no blank pages in the middle that only contain a figure or a table. """ + "\n" + UNIVERSAL_NO_LEAKAGE_PROMPT - methods/prompts/literature_review_agent.py (2993 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.prompt_utils import UNIVERSAL_NO_LEAKAGE_PROMPT literature_review_agent_writter_prompt = """Role: Senior AI Researcher. Task: Write the introduction and related work section of a paper. You will be given a 'template.tex', this is the initial skeleton we outlined for you. Your job is to fill in two sections: Introduction and Related Work. Leave all the other sections untouched. INPUTS: * 'intro_related_work_plan': This is your PRIMARY guide for structure and arguments. * 'project_idea' and 'project_experimental_log': Use them to ensure the Intro accurately frames the technical contribution and results. * 'citation_checklist': This includes the citation keys that you should use when citing relevant papers. * 'collected_papers': These are all the relevant papers we collect for you for citation purpose. YOU MUST ONLY CITE THE GIVEN 'collected_papers', DO NOT cite new papers other than the given papers. CITATION REQUIREMENTS: - You have access to the abstract of {paper_count} collected papers. - You MUST cite at least {min_cite_paper_count} of them across the introduction and related work sections. - Introduction: Cite key statistics, foundational models (CLIP, etc.), and broad problem statements. - Related Work: Do deep comparative citations. Group distinct works (e.g., "Several methods [A, B, C]..."). - Ensure every \\cite{{key}} corresponds exactly to a key in 'citation_checklist'. - CRITICAL TIMELINE RULE: Do not treat any papers published after {cutoff_date} as prior baselines to beat. Treat them strictly as concurrent work. - CRITICAL EVALUATION RULE: Do not claim our method beats or achieves State-of-the-Art over a specific cited paper UNLESS that paper is explicitly evaluated against in 'project_experimental_log'. Frame other recent papers strictly as concurrent, orthogonal, or conceptual work. - You need to return the full code for the new 'template.tex', where the two empty sections (Introduction and Related Work) are now fille in, while all the other code (packages, styles, and other sections) are identical to the original 'template.tex'. IMPORTANT NOTE: - DO NOT change '\\usepackage[capitalize]{{cleveref}}' into '\\usepackage[capitalize]{{cleverref}}', as there's no 'cleverref.sty'. OUTPUT Format: You must return the code for the updated 'template.tex', Make sure to wrap the code with ```latex content```. """ + "\n" + UNIVERSAL_NO_LEAKAGE_PROMPT - methods/prompts/outline_agent.py (11845 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.prompt_utils import UNIVERSAL_NO_LEAKAGE_PROMPT outline_agent_system_prompt = ( """You are a senior AI researcher drafting a paper for a top-tier conference (e.g., NeurIPS, ICML, CVPR, ICLR). Your task is to convert the provided methodology and experimental logs into a detailed, venue-compliant paper outline. You must output a single JSON object. Your inputs are: 1. `idea.md`: A detailed summary of the methodology, core contributions, and theoretical framework. 2. `experimental_log.md`: A summary of experimental results, including raw data points, ablation studies, and performance metrics. 3. `template.tex`: The target structure. You must use the section commands (e.g., `\\section{{...}}`) found here as your primary skeleton. 4. `conference_guidelines.md`: Formatting rules, specific page limits (for word count calculation), and mandatory sections. ### Processing Directives Global Instruction: Do not analyze inputs in isolation. You must synthesize information across all provided documents for every step. #### Directive 1: Plotting & Visualization Plan Synthesize `experimental_log.md` and `idea.md` to identify the most compelling evidence. * Determine which figures are essential to visually prove the hypothesis (e.g., convergence rates, qualitative visual comparisons). * The `plot_type` MUST be exactly "plot" or "diagram". If it is a plot, specify the specific chart type (e.g., Radar Chart) inside the `objective`. * The `data_source` MUST be exactly "idea.md", "experimental_log.md", or "both". * Determine the ideal `aspect_ratio` for each figure. The aspect_ratio MUST be exactly one of: "1:1", "1:4", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9". * The `figure_id` MUST be a semantically meaningful string identifier summarizing the plot contents, like "fig_framework_overview" or "fig_ablation_study_parameter_sensitivity". It MUST NOT contain the word "Figure". * Output Focus: Create an array of objects for the `plotting_plan` key. #### Directive 2: Research Graph & Investigation Strategy (Intro & Related Work) Provide search instructions for a downstream literature review agent to build a Research Graph. Do not write the actual paper content. Prevent Citation Overlap: Strictly separate the scope of the Introduction from Related Work to ensure the agent searches for different tiers of literature. * Introduction: Focuses on macro-level context (foundational papers, surveys). * Related Work: Focuses on micro-level technical comparisons (recent SOTA baselines, benchmarks). * Introduction Strategy (Macro-Level Context, 10-20 papers): * Hypotheses: Define the "Hook" (broad context) and "Problem Gap" to be verified. CRITICAL: Strictly scope the problem gap and claims to match the specific datasets and evaluations present in `experimental_log.md`. Do not over-claim generalization. * Search Directions: Provide 3-5 specific queries to find: 1. Papers establishing the real-world impact or urgency of the problem gap. 2. Good survey or review papers on the topic. 3. 3-5 Foundational papers that established the sub-field. * Related Work Strategy (Micro-Level Technical Baselines, 30-50 papers): * Divide the field into 2-4 distinct methodology clusters that directly compete with or precede our approach. * For each cluster, define: 1. Methodology Cluster Name: The technical category. 2. SOTA Investigation: Instructions to find recent papers for conceptual context. CRITICAL TIMELINE RULE: Do not instruct searches for any papers published after {cutoff_date}. Furthermore, do NOT instruct the search for new "competitors" to beat if they are not exclusively in `experimental_log.md`. 3. Limitation Hypothesis: The suspected failure point of these competing methods, based on `idea.md`. 4. Limitation Search Queries: Highly specific, narrow queries to find papers documenting these exact limitations. 5. The Bridge: How our proposed method resolves this specific limitation. * Output Focus: Populate the `intro_related_work_plan` key. #### Directive 3: Section Writing Plan & Sizing Constraints Outline the remaining sections (Abstract, Methodology, Experiments, Conclusion, Appendix) into a detailed structural plan. * Structural Hierarchy: If Subsection X.1 is created, X.2 is mandatory. Do not create orphaned subsections. Omit subsections entirely if a section does not require division. * Content Specificity: Explicitly reference source materials. * *Avoid:* "Describe the model." * *Require:* "Formalize the Temporal-Aware Attention mechanism using Eq. 3 from idea.md." * Mandatory Citations (`citation_hints`): You must provide targeted citation hints for all external dependencies. Every hint must point to a single, unambiguous canonical paper. * Required Coverage (EXHAUSTIVE): You MUST explicitly create a targeted `citation_hints` query for EVERY SINGLE dataset, optimizer, metric, and foundational architecture/model you mention, no matter how ubiquitous or obvious it seems (e.g., AdamW, ResNet, ImageNet, CLIP, Transformer, LLaMA, GPT, LLaVA). If it is in the `experimental_log.md` or `idea.md`, it MUST have a citation hint. 1. All baseline methods compared against. 2. All datasets evaluated on. 3. All standard metrics utilized. 4. All foundational algorithms, architectures (e.g., ResNet, Transformer), foundational models (e.g., LLMs, VLMs, Diffusion models), optimizers (e.g., AdamW), or frameworks built upon. * Format Constraint & Anti-Hallucination Rule: If you know the exact author and title, use "Author (Exact Paper Title)". DO NOT guess or hallucinate authors. If you do not know the exact author, use this format: "research paper or technical report introducing '[Exact Model/Dataset/Metric Name]'". * Output Focus: Populate the `section_plan` key. Guidelines on Scientific Depth & Mathematical Rigor: - Grounded Formalization: Propose explicit subsections for rigorous mathematical formulations (e.g., loss functions, core algorithms, theoretical proofs). You must base these strictly on `idea.md` and `experimental_log.md`; do not instruct the writing agent to include hallucinated variables or unsupported math. ### Strict Output Format (JSON) You must output a single, valid JSON object with the following three top-level keys: "plotting_plan", "intro_related_work_plan", and "section_plan". Example Output: ```json {{ "plotting_plan": [ {{ "figure_id": "fig_teaser_fig_cross_modal_alignment_performance", "title": "Teaser: Cross-Modal Alignment Performance", "plot_type": "plot", "data_source": "experimental_log.md", "objective": "Visual summary (Radar Chart) demonstrating that our method achieves SOTA balance across 5 metrics.", "aspect_ratio": "16:9" }} ], "intro_related_work_plan": {{ "introduction_strategy": {{ "hook_hypothesis": "Video-LLMs are currently the dominant paradigm for short clips.", "problem_gap_hypothesis": "Context window limits prevent scaling to >5s videos efficiently.", "search_directions": [ "Find highly cited papers establishing the real-world impact of context limits in video generation", "Search for published 'long-context video generation' surveys", "Identify foundational papers establishing causal video generation" ] }}, "related_work_strategy": {{ "overview": "Investigate three specific paradigms to build a graph proving the necessity of our Sliding-Window approach.", "subsections": [ {{ "subsection_title": "2.1 Autoregressive Video Generation", "methodology_cluster": "Discrete Tokenization & Transformers", "sota_investigation_mission": "Identify the current SOTA autoregressive models from 2024-2025. Determine their maximum stable generation length.", "limitation_hypothesis": "These models suffer from 'drift' or 'error propagation' because they lack bidirectional context.", "limitation_search_queries": [ "Autoregressive video generation error propagation metrics", "Causal masking limitations in temporal video transformers" ], "bridge_to_our_method": "Our method introduces bidirectional blocks to fix the hypothesized drift issue." }}, {{ "subsection_title": "2.2 Diffusion-Based Editing Frameworks", "methodology_cluster": "DDIM Inversion & Cross-Attention", "sota_investigation_mission": "Find recent papers using DDIM inversion for editing. Identify the standard benchmarks they use.", "limitation_hypothesis": "They fail at large structural changes because cross-attention maps are too rigid.", "limitation_search_queries": [ "DDIM inversion failure cases large motion", "Cross-attention control rigidity video editing" ], "bridge_to_our_method": "Our Flow-Guided Attention allows for spatial deformation, addressing rigidity." }} ] }} }}, "section_plan": [ {{ "section_title": "Abstract", "subsections": [ {{ "subsection_title": "Abstract Content", "content_bullets": [ "Briefly state the problem of temporal inconsistency.", "Introduce the proposed method.", "Highlight key results." ], "citation_hints": [] }} ] }}, {{ "section_title": "3. Methodology", "subsections": [ {{ "subsection_title": "3.1 Temporal-Aware Attention Mechanism", "content_bullets": ["Define the query-key matching logic", "Explain the masking strategy"], "citation_hints": [ "Vaswani et al. (Attention Is All You Need)", "research paper or technical report introducing 'FlashAttention-2'" ] }}, {{ "subsection_title": "3.2 Optimization Objective", "content_bullets": ["Detail the loss function", "Discuss regularization terms"], "citation_hints": [] }} ] }}, {{ "section_title": "4. Experiments", "subsections": [ {{ "subsection_title": "4.1 Experimental Setup", "content_bullets": ["Implementation details", "Hyperparameters and datasets used"], "citation_hints": [ "research paper or technical report introducing 'WebVid-10M'", "Paszke et al. (PyTorch: An Imperative Style, High-Performance Deep Learning Library)", "research paper or technical report introducing 'AdamW optimizer'", "research paper or technical report introducing 'Jaccard Index metric'" ] }}, {{ "subsection_title": "4.2 Main Results", "content_bullets": ["Comparison with Baselines", "Quantitative Analysis"], "citation_hints": [ "Ho et al. (Denoising Diffusion Probabilistic Models)", "research paper or technical report introducing 'AVSegFormer baseline'" ] }} ] }} ] }} ``` """ + "\n" + UNIVERSAL_NO_LEAKAGE_PROMPT ) - methods/prompts/section_writing_agent.py (5243 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from utils.prompt_utils import UNIVERSAL_NO_LEAKAGE_PROMPT section_writing_agent_prompt = """Role: Senior AI Researcher. Task: Complete a research paper by writing the missing sections in a LaTeX template. You will be given a 'template.tex' file where some sections (e.g., Introduction, Related Work) are already written, and others are empty or missing. Your job is to generate the LaTeX code for the missing sections only, based on the provided 'outline.json', and merge them into the final document. INPUTS: * 'outline.json': Your MASTER PLAN. Defines section hierarchy, points to cover, and which papers to consider citing (`citation_candidates`). * 'idea.md': Technical details of the Methodology. * 'experimental_log.md': Raw data for tables and qualitative analysis for text. * 'citation_map.json': A Reference Library containing the BibTeX keys, titles, and abstracts of papers. * 'conference_guidelines.md': Formatting rules. * 'figures_list': Available figure files. CRITICAL INSTRUCTIONS: 1. Existing Content Preservation: - DO NOT modify the text, style, or content of sections that are already filled in 'template.tex'. - Come up with a good title if it's missing, fill in the author names if missing. - Keep the preamble (packages) exactly as is. 2. Data & Tables: - You are responsible for creating LaTeX tables. - Extract numerical data directly from 'experimental_log.md'. - Use the `booktabs` package format (\\toprule, \\midrule, \\bottomrule). - Do not hallucinate numbers. Use the exact values provided in the log. - Make sure all tables appear before the Conclusion section, unless it's placed in an Appendix. 3. Citations: - The 'outline.json' provides a list of `citation_candidates` for specific subsections. - You MUST use the exact keys found in `citation_map.json` (e.g., `\\cite{Hu2021LoraLowrank}`). - Content Enrichment: Read the `abstract` provided in `citation_map.json` for the papers you are citing. Use this context to write accurate, specific sentences about those works. 4. Writing Content: - Write the missing sections following the 'outline.json' structure. - Use formal mathematical equations, notations, and definitions where appropriate and directly supported by the idea/log. DO NOT hallucinate incorrect or overly complex math just for the sake of it; keep it accurate and grounded in the provided context. Avoid overly colloquial summaries. - Always provide detailed ablation studies and qualitative analysis of the experimental results, what works and what doesn't, and why. - Nice to have: discuss the limitation and future work at the end. - If you want to put anything in Appendix, make sure the 'Appendix' section appears after the 'References' section, on a fresh new page. 5. Figures and Visual Fidelity: - You are being provided with the actual image files of the figures. You MUST describe them faithfully and accurately. DO NOT hallucinate interpretations that contradict the visual evidence in the plots. - Make sure to use ALL of the figures provided in 'figures_list'. Note: figures are stored in the 'figures/' subdirectory. IMPORTANT: use the exact filenames including their extensions (e.g., .png) in your \\includegraphics commands. - DO NOT merge or group multiple figures into one for display. - If the paper is 2-column format, try displaying figures in single-column mode (\\begin{figure}`) unless they are very wide. - Ensure that all figures are correctly referenced in the text. - Make sure all figures appears before the Conclusion section, unless it's placed in an Appendix. - You can refine the captions if necessary. - Don't include "Figure x" in the caption text, the LaTeX template will handle the figure numbering. 6. Style: - Adopt the tone of a top-tier ML conference paper: dense, objective, and technical. - Ensure your new LaTeX code matches the indentation and spacing style of the `template.tex`. Don't change the given style. OUTPUT FORMAT: - Return the full code for the completed 'template.tex'. - The sections that were previously empty should now be filled. - The sections that were previously filled should remain mostly untouched, only adjust for consistency purposes. - Wrap the code with ```latex content```. IMPORTANT NOTE: - DO NOT change '\\usepackage[capitalize]{{cleveref}}' into '\\usepackage[capitalize]{{cleverref}}', as there's no 'cleverref.sty'. - Ensure the LaTeX code compiles without errors, e.g. all the begin and end statements match correctly (e.g., \\begin{figure*} must be closed with \\end{figure*}, not \\end{figure}). """ + "\n" + UNIVERSAL_NO_LEAKAGE_PROMPT - paper_writing_cli.py (6415 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import sys import os import shutil from datetime import datetime def main(): parser = argparse.ArgumentParser( description="CLI for generating a single paper from the Paper Orchestra pipeline." ) parser.add_argument( "--raw_materials_dir", required=True, help="Directory containing the raw materials (e.g. idea and experimental log).", ) parser.add_argument( "--output_dir", default=None, help="Directory to output the generated paper. If not specified, an automatic folder is created.", ) parser.add_argument( "--latex_template_dir", required=True, help="Directory containing the LaTeX template (e.g. cvpr2025/iclr2025).", ) parser.add_argument( "--idea_filename", default="idea_sparse.md", help="Filename of the idea document within raw_materials. Default: idea_sparse.md", ) parser.add_argument( "--experimental_log_filename", default="experimental_log.md", help="Filename of the experimental log within raw_materials. Default: experimental_log.md", ) parser.add_argument( "--writer_model_name", default="gemini-3.1-pro-preview", help="LLM for writer and literature agents.", ) parser.add_argument( "--reflection_model_name", default="gemini-3.1-pro-preview", help="LLM for reflection agents.", ) parser.add_argument( "--research_cutoff", default=None, help="Year-Month cutoff date for literature review (e.g., 2024-11). If not provided, defaults to current Year-Month.", ) parser.add_argument( "--use_plotting", type=lambda x: (str(x).lower() in ["true", "1", "yes"]), default=False, help="Enable plotting agent workflow to generate figures from code.", ) parser.add_argument( "--plotting_model_name", default="gemini-3.1-pro-preview", help="LLM for the plotting agent.", ) parser.add_argument( "--image_model_name", default="gemini-3-pro-image-preview", help="Vision model if required by plotting agent.", ) parser.add_argument( "--plotting_max_critic_rounds", type=int, default=3, help="Maximum rounds of self-correction in the plotting agent.", ) args = parser.parse_args() if args.research_cutoff is None: args.research_cutoff = datetime.now().strftime("%Y-%m") if not os.path.exists(args.raw_materials_dir): print( f"Error: Raw materials directory '{args.raw_materials_dir}' does not exist." ) sys.exit(1) if args.output_dir: base_dir = args.output_dir else: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") base_dir = f"./paper_output_{timestamp}" os.makedirs(base_dir, exist_ok=True) target_raw_materials = os.path.join(base_dir, "raw_materials") if os.path.abspath(target_raw_materials) != os.path.abspath(args.raw_materials_dir): if os.path.exists(target_raw_materials): shutil.rmtree(target_raw_materials) shutil.copytree(args.raw_materials_dir, target_raw_materials) if not args.use_plotting: has_figures = False figures_dir = os.path.join(args.raw_materials_dir, "figures") if os.path.exists(figures_dir): for f in os.listdir(figures_dir): if f.lower().endswith((".pdf", ".png", ".jpg", ".jpeg")): has_figures = True break if not has_figures: print("\n" + "=" * 80) print( f"⚠️ WARNING: No figures (.pdf, .png, .jpg) detected in the {figures_dir} directory." ) print( "Since you are running WITHOUT plotting, we are using the provided materials directly." ) print( f"Please make sure you provide all necessary figures inside {figures_dir}." ) print("If figures are missing, compilation may fail.") print( f"If you don't have figures, you can use the plotting agent to generate them by running with --use_plotting." ) print("=" * 80 + "\n") if args.use_plotting: from methods.paper_writer_with_plotting import write_single_paper print(f"🚀 Starting Paper Generation (WITH PLOTTING) in {base_dir}") write_single_paper( base_dir=base_dir, latex_template_dir=args.latex_template_dir, idea_filename=args.idea_filename, experimental_log_filename=args.experimental_log_filename, writer_model_name=args.writer_model_name, reflection_model_name=args.reflection_model_name, plotting_model_name=args.plotting_model_name, image_model_name=args.image_model_name, research_cutoff=args.research_cutoff, plotting_max_critic_rounds=args.plotting_max_critic_rounds, ) else: from methods.paper_writer import write_single_paper provided_figures_dir = os.path.join(args.raw_materials_dir, "figures") print( f"🚀 Starting Paper Generation (WITHOUT PLOTTING), with figures stored in {provided_figures_dir}" ) write_single_paper( base_dir=base_dir, latex_template_dir=args.latex_template_dir, idea_filename=args.idea_filename, experimental_log_filename=args.experimental_log_filename, writer_model_name=args.writer_model_name, reflection_model_name=args.reflection_model_name, research_cutoff=args.research_cutoff, ) if __name__ == "__main__": main() - paper_writing_cli.sh (997 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MY_ENV=paper_orchestra PAPER_ID="cvpr2025_0b62029e18" # Switch this to the paper id you want to write PYTHONUNBUFFERED=1 nohup conda run --no-capture-output -n $MY_ENV python paper_writing_cli.py \ --raw_materials_dir datasets/cvpr2025/papers/${PAPER_ID}/raw_materials \ --latex_template_dir templates/cvpr2025 \ --output_dir ./paper_output_${PAPER_ID} \ --use_plotting true \ > ./paper_writing_${PAPER_ID}.log 2>&1 & - README.md (4967 bytes)
# <div align="center">🎻 PaperOrchestra: A Multi-Agent Framework for Automated AI Research Paper Writing</div> <div align="center">Yiwen Song<sup>1</sup>, Yale Song<sup>1</sup>, Tomas Pfister<sup>1</sup>, and Jinsung Yoon<sup>1</sup></div> <div align="center"><sup>1</sup>Google Cloud AI Research</div> <br><br> PaperOrchestra is a multi-agent framework for automated AI research paper writing. It flexibly transforms unconstrained pre-writing materials (ideas and experimental logs) into submission-ready LaTeX manuscripts, including comprehensive literature synthesis and generated visuals, such as plots and conceptual diagrams. Acting like an orchestrated team of specialized agents, it handles outline generation, literature review, section writing, content refinement, and plotting to close the gap between raw research materials and submission ready papers. <div align="center"> <img src="assets/overview.png" alt="PaperOrchestra Overview" width="90%"/> </div> ## Key Features - **Multi-Agent Pipeline**: Orchestrates specialized agents (Outline, Literature Review, Section Writing, Content Refinement, Plotting) for end-to-end paper generation. - **Comprehensive Literature Synthesis**: Automatically searches, prioritizes, and integrates relevant literature into the manuscript. - **Visuals Generation**: Generates plots and supports the inclusion of conceptual diagrams. - **LaTeX Automation**: Produces submission-ready LaTeX manuscripts adhering to target conference templates. ## Installation 1. **Environment Setup**: The project relies on a Conda environment named `paper_orchestra`. You can set up a similar environment with Python 3.11 and install the required dependencies. ```bash conda create -n paper_orchestra python=3.11 conda activate paper_orchestra pip install -r requirements.txt ``` 2. **Environment Variables**: Create a `.env` file in the root of the project and add your API keys: ```env # Add keys if you have them, otherwise leave them empty # You need to specify related keys if you want to use the model # For example, if you want to use Gemini models, you need to specify either VERTEX_AI_PROJECT and VERTEX_AI_LOCATION, or GEMINI_API_KEY # If you want to use OpenAI models, you need to specify OPENAI_API_KEY SEMANTIC_SCHOLAR_API_KEY=your_semantic_scholar_api_key OPENAI_API_KEY=your_openai_api_key VERTEX_AI_PROJECT=your_vertex_ai_project VERTEX_AI_LOCATION=your_vertex_ai_location GEMINI_API_KEY=your_gemini_api_key # if you want to run demo and send emails to yourself at the end, set these 2 fields as well SMTP_EMAIL=your_email SMTP_PASSWORD=your_password ``` ## Dataset The dataset used for paper generation is **not included** in this code repository. It will be released separately at a later date. Once available, you should download the dataset and place it under the `datasets/` directory. ## Usage You can run the paper generation pipeline using the provided bash script or directly via the Python CLI. ### Using the Bash Script ```bash ./paper_writing_cli.sh ``` ### Using the Python CLI directly ```bash python paper_writing_cli.py \ --raw_materials_dir /path_to_your_raw_materials/ \ --latex_template_dir /path_to_your_latex_template/ \ --output_dir /path_to_your_output_folder/ \ --use_plotting true ``` ### Key Arguments * `--raw_materials_dir`: (Required) Directory containing the raw materials (e.g., idea and experimental log). * `--latex_template_dir`: (Required) Directory containing the LaTeX template (e.g., `templates/cvpr2025`). * `--output_dir`: Directory to output the generated paper. If not specified, an automatic folder with a timestamp is created. * `--use_plotting`: Enable the plotting agent workflow to generate figures from code (Default: `False`). * `--writer_model_name`: LLM for writer and literature agents. * `--reflection_model_name`: LLM for reflection agents. ### Interactive Demo We provide an interactive Streamlit demo for the paper generation pipeline. For instructions on how to run and use it, please refer to the [README.md](frontend/README.md) in the `frontend` directory. ## Project Structure * `autoraters/`: Contains scripts for automated evaluation. * `methods/`: Core logic for agents and pipeline execution. * `templates/`: LaTeX templates for target conferences, you can add your own template by creating a new subfolder under this directory. * `utils/`: Utility functions for API calls, PDF handling, etc. ## Citation If you find this repo or our paper helpful, please cite it as follows: ```bibtex @article{song2026paperorchestra, title={PaperOrchestra: A Multi-Agent Framework for Automated AI Research Paper Writing}, author={Song, Yiwen and Song, Yale and Pfister, Tomas and Yoon, Jinsung}, journal={arXiv preprint arXiv:2604.05018}, year={2026} } ``` # Disclaimer This is not an officially supported Google product. - requirements.txt (165 bytes)
google-genai joblib json_repair matplotlib numpy openai opencv-python pydantic PyMuPDF pymupdf4llm pyOpenSSL pypdf python-dotenv requests rich streamlit thefuzz tqdm
- templates/cvpr2025/guidelines.md (8799 bytes)
# CVPR 2025 Paper Writing Guidelines Below are the instructions to help you with the latex paper writing process for CVPR 2025. ## Language All manuscripts must be in English. ## Submission Policies & Anonymity CVPR reviewing is double-blind. Authors must not know the names of the reviewers, and reviewers must not know the names of the authors. * **No Identifying Information:** Do not provide information that may identify the authors in the acknowledgments (e.g., co-workers and grant IDs) or in the supplementary material. * **Links:** Do not provide links to websites that identify the authors. * **GitHub:** Links to GitHub must be anonymous. * **Self-Citation:** If you need to cite any of your own papers that are being submitted concurrently, include anonymized versions in the supplementary material and cite them as anonymous. Do not use the words "my" or "our" when citing previous work; treat your own prior work as if it were written by someone else (e.g., "Smith et al. showed..." rather than "We showed..."). ## Paper length Papers, excluding the references section, must be no longer than eight pages in length. The references section will not be included in the page count, and there is no limit on the length of the references section. For example, a paper of eight pages with two pages of references would have a total length of 10 pages. Overlength papers, or papers where margins and formatting are deemed significantly altered, will simply not be reviewed. ## Mathematics Please number all of your sections and displayed equations as in these examples: ```latex \begin{equation} E = m\cdot c^2 \label{eq:important} \end{equation} ``` and ```latex \begin{equation} v = a\cdot t. \label{eq:also-important} \end{equation} ``` ## Formatting your paper All text must be in a two-column format. The total allowable size of the text area is $6\frac78$ inches (17.46 cm) wide by $8\frac78$ inches (22.54 cm) high. Columns are to be $3\frac14$ inches (8.25 cm) wide, with a $\frac{5}{16}$ inch (0.8 cm) space between them. The main title (on the first page) should begin 1 inch (2.54 cm) from the top edge of the page. The second and following pages should begin 1 inch (2.54 cm) from the top edge. On all pages, the bottom margin should be $1\frac{1}{8}$ inches (2.86 cm) from the bottom edge of the page for $8.5 \times 11$-inch paper; for A4 paper, approximately $1\frac{5}{8}$ inches (4.13 cm) from the bottom edge of the page. ### Type style and fonts Wherever Times is specified, Times Roman may also be used. If neither is available on your word processor, please use the font closest in appearance to Times to which you have access. ### Main Title Center the title $1\frac{3}{8}$ inches (3.49 cm) from the top edge of the first page. The title should be in Times 14-point, boldface type. Capitalize the first letter of nouns, pronouns, verbs, adjectives, and adverbs; do not capitalize articles, coordinate conjunctions, or prepositions (unless the title begins with such a word). The ABSTRACT and MAIN TEXT are to be in a two-column format. ### Main Text Type main text in 10-point Times, single-spaced. Do NOT use double-spacing. All paragraphs should be indented 1 pica (approx.~$\frac{1}{6}$ inch or 0.422 cm). Make sure your text is fully justified---that is, flush left and flush right. Please do not place any additional blank lines between paragraphs. Figure and table captions should be 9-point Roman type as in \cref{fig:onecol,fig:short}. Short captions should be centred. \noindent Callouts should be 9-point Helvetica, non-boldface type. Initially capitalize only the first word of section titles and first-, second-, and third-order headings. #### First-order Headings For example, {\large \bf 1. Introduction} should be Times 12-point boldface, initially capitalized, flush left, with one blank line before, and one blank line after. #### Second-order Headings For example, { \bf 1.1. Database elements} should be Times 11-point boldface, initially capitalized, flush left, with one blank line before, and one after. If you require a third-order heading (we discourage it), use 10-point Times, boldface, initially capitalized, flush left, preceded by one blank line, followed by a period and your text on the same line. ### Footnotes Please use \footnote{This is what a footnote looks like. It often distracts the reader from the main flow of the argument.} sparingly. Indeed, try to avoid footnotes altogether and include necessary peripheral observations in the text (within parentheses, if you prefer, as in this sentence). If you wish to use a footnote, place it at the bottom of the column on the page on which it is referenced. Use Times 8-point type, single-spaced. ### Cross-references For the benefit of author(s) and readers, please use the ```latex {\small\begin{verbatim} \cref{...} \end{verbatim}} ``` command for cross-referencing to figures, tables, equations, or sections. This will automatically insert the appropriate label alongside the cross-reference as in this example: ```latex \begin{quotation} To see how our method outperforms previous work, please see \cref{fig:onecol} and \cref{tab:example}. It is also possible to refer to multiple targets as once, \eg~to \cref{fig:onecol,fig:short-a}. You may also return to \cref{sec:formatting} or look at \cref{eq:also-important}. \end{quotation} ``` If you do not wish to abbreviate the label, for example at the beginning of the sentence, you can use the ```latex {\small\begin{verbatim} \Cref{...} \end{verbatim}} command. Here is an example: \begin{quotation} \Cref{fig:onecol} is also quite important. \end{quotation} ``` ## Example Code for Tables ```latex \begin{table} \centering \begin{tabular}{@{}lc@{}} \toprule Method & Frobnability \\ \midrule Theirs & Frumpy \\ Yours & Frobbly \\ Ours & Makes one's heart Frob\\ \bottomrule \end{tabular} \caption{Results. Ours is better.} \label{tab:example} \end{table} ``` ## Example Code for Figures ```latex \begin{figure}[t] \centering \fbox{\rule{0pt}{2in} \rule{0.9\linewidth}{0pt}} %\includegraphics[width=0.8\linewidth]{egfigure.eps} \caption{Example of caption. It is set in Roman so that mathematics (always set in Roman: $B \sin A = A \sin B$) may be included without an ugly clash.} \label{fig:onecol} \end{figure} \begin{figure*} \centering \begin{subfigure}{0.68\linewidth} \fbox{\rule{0pt}{2in} \rule{.9\linewidth}{0pt}} \caption{An example of a subfigure.} \label{fig:short-a} \end{subfigure} \hfill \begin{subfigure}{0.28\linewidth} \fbox{\rule{0pt}{2in} \rule{.9\linewidth}{0pt}} \caption{Another example of a subfigure.} \label{fig:short-b} \end{subfigure} \caption{Example of a short caption, which should be centered.} \label{fig:short} \end{figure*} ``` ## Illustrations, graphs, and photographs All graphics should be centered. In \LaTeX, avoid using the \texttt{center} environment for this purpose, as this adds potentially unwanted whitespace. Instead use ```latex {\small\begin{verbatim} \centering \end{verbatim}} ``` at the beginning of your figure. Please ensure that any point you wish to make is resolvable in a printed copy of the paper. Resize fonts in figures to match the font in the body text, and choose line widths that render effectively in print. When placing figures in \LaTeX, it's almost always best to use \verb+\includegraphics+, and to specify the figure width as a multiple of the line width as in the example below ```latex {\small\begin{verbatim} \usepackage{graphicx} ... \includegraphics[width=0.8\linewidth] {myfile.pdf} \end{verbatim}} ``` ## Color If you use color in your plots, please keep in mind that a significant subset of reviewers and readers may have a color vision deficiency; red-green blindness is the most frequent kind. Hence avoid relying only on color as the discriminative feature in plots (such as red \vs green lines), but add a second discriminative feature to ease disambiguation. ## Important Rules * DO NOT change `\usepackage[capitalize]{cleveref}` into `\usepackage[capitalize]{cleverref}`, as there's no `cleverref.sty`. * Keep the existing packages in the template unchanged. * For consistent compilation across environments, all figures will reside in the `figures` directory. Always include the `figures/` prefix in your `\includegraphics` command. Failing to include this prefix (e.g., using `figure_1.png` instead of `figures/figure_1.png`) will result in figure loading errors during the build process. * Always use the following author information placeholder in your writeup: * Ambitious AI Researcher, AI Research Institute, 123 AI Avenue, ML City, [email protected] - templates/iclr2025/guidelines.md (4936 bytes)
# ICLR 2025 Paper Writing Guidelines Below are the instructions to help you with the LaTeX paper writing process for ICLR 2025. ## General Setup * **Document Class:** `\documentclass{article}` * **Required Packages:** ```latex \usepackage{iclr2025_conference,times} \usepackage{hyperref} \usepackage{url} % Optional but recommended for math \input{math_commands.tex} ``` * **Style Files:** You must use `iclr2025_conference.sty` and `iclr2025_conference.bst`. ## Anonymity (Crucial) * ICLR is double-blind. **Do not** include real author names or affiliations in the submission version, use "Ambitious AI Researcher" instead. * Use the third person when citing your own prior work (e.g., "Earlier work by Smalle (2024)..." instead of "In our earlier work..."). * You may post your paper on arXiv, but do not explicitly refer to the arXiv version in the paper to maintain anonymity. ## Paper Length * **Main Text:** Strict upper limit of **10 pages**. * **Citations:** Unlimited additional pages. * **Ethics & Reproducibility Statements:** Optional. Do not count toward the page limit (max 1 page each). * **Acknowledgments:** Do not count toward the page limit. ## Abstract The abstract must be limited to **one paragraph**. * **Margins:** Indented 1/2 inch (3 picas) on both left and right-hand margins. * **Font:** 10-point type, vertical spacing of 11-points. * **Title:** The word `\textsc{Abstract}` must be centered, in small caps, and in point size 12. * **Spacing:** Two line spaces precede the abstract. ## Formatting Your Paper The text must be confined within a rectangle 5.5 inches (33 picas) wide and 9 inches (54 picas) long. The left margin is 1.5 inch (9 picas). * **Font:** Times New Roman (via `times` package) is preferred. * **Size:** 10-point type with 11-point vertical spacing. * **Paragraphs:** Separated by 1/2 line space, with **no indentation**. * **Paper Title:** 17-point, small caps, left-aligned. ### Headings * **First Level:** Small caps (`\textsc`), flush left, 12-point. One line space before, 1/2 after. * **Second Level:** Small caps, flush left, 10-point. One line space before, 1/2 after. * **Third Level:** Small caps, flush left, 10-point. One line space before, 1/2 after. ## Citations and References Use the `natbib` package (loaded automatically by the style file). * **Narrative citation:** Use `\citet{Gomu2024}` $\rightarrow$ "Gomu et al. (2024)". * **Parenthetical citation:** Use `\citep{Gomu2024}` $\rightarrow$ "(Gomu et al., 2024)". * **Bibliography:** ```latex \bibliography{references} \bibliographystyle{iclr2025_conference} ``` ## Figures and Tables * All artwork must be neat, clean, and legible. * **Captions:** Lower case (except for first word and proper nouns). * **Placement:** * **Figures:** Caption appears **after** (below) the figure. * **Tables:** Caption appears **before** (above) the table. ### Example Code for Figures ```latex \begin{figure}[h] \begin{center} %\framebox[4.0in]{$\;$} \fbox{\rule[-.5cm]{0cm}{4cm} \rule[-.5cm]{4cm}{0cm}} \end{center} \caption{Sample figure caption.} \end{figure} ``` ### Example Code for Tables ```latex \begin{table}[t] \caption{Sample table title} \label{sample-table} \begin{center} \begin{tabular}{ll} \multicolumn{1}{c}{\bf PART} &\multicolumn{1}{c}{\bf DESCRIPTION} \\ \hline \\ Dendrite &Input terminal \\ Axon &Output terminal \\ Soma &Cell body (contains cell nucleus) \\ \end{tabular} \end{center} \end{table} ``` ## Math Notation * ICLR encourages standardized notation using `\input{math_commands.tex}` (based on the Deep Learning book notation). * Variables: Italics ($a$, $\va$, $\mA$). * Random Variables: Roman ($P(\ra)$). * Sets: Blackboard bold ($\R$, $\sA$). * Functions: Standard LaTeX ($f(\vx)$). ## Optional Statements These should appear at the end of the main text, before references. 1. **Ethics Statement:** (Optional, max 1 page) Address potential concerns (human subjects, sensitive data, bias, etc.). 2. **Reproducibility Statement:** (Optional, max 1 page) Reference code, proofs, or data processing steps to ensure reproducibility. ## Important Rules * **Formatting:** Do not modify the width or length of the text rectangle or font sizes. * **Line Numbers:** Do not refer to line numbers in your text (the style file adds them automatically for review). * **Appendix:** You may include an appendix at the very end of the main PDF (after references). Use `\appendix` to start this section. * **Figures Directory:** For consistent compilation, all figures will reside in the `figures` directory. Always include the `figures/` prefix in your `\includegraphics` command (e.g., `figures/figure_1.pdf`). * **Author Information:** Always use the following author information placeholder in your writeup: * Ambitious AI Researcher, AI Research Institute, 123 AI Avenue, ML City, [email protected] - utils/common_utils.py (1551 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import os import os.path as osp def create_log_folder( prefix: str = "", date_str: str = "", log_dir: str = "./logs/paper_writing", ): now = datetime.datetime.now() timestamp_str = now.strftime("%Y_%m_%d_%H_%M_%S") log_dir_name = f"log_{timestamp_str}" if prefix != "": log_dir_name = f"{prefix}_" + log_dir_name if date_str == "": date_str = now.strftime("%Y%m%d") log_path = os.path.join(log_dir, date_str, log_dir_name) os.makedirs(log_path, exist_ok=True) print(f"Generated Log Directory Name: **{log_dir_name}**") print(f"Full Log Path: **{log_path}**") return log_path def load_md_file(md_file_path: str): if not osp.exists(md_file_path): print( f"Error! Markdown file not found at '{md_file_path}', use empty string..." ) return None with open(md_file_path, "r") as f: prompt_content = f.read() return prompt_content - utils/content_parsing_utils.py (1756 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from .gemini_utils import call_gemini_with_text_prompt extract_title_prompt_template = """You are an expert bibliographic parser. Extract the full research paper title from the messy citation text below. ## Rules: 1. Return ONLY the title text. No authors, years, venues, or labels. 2. HANDLING HYPHENS (Important): - ONLY remove a hyphen if it clearly splits a single word due to a line break (e.g., "Net- work" -> "Network"). - PRESERVE hyphens in compound words (e.g., keep "Text-to-Image", "Zero-shot"). 3. FIX MERGED WORDS (Important): - If spaces are missing between words (common in PDF extraction), split them. - Examples: Change "Imagesearch" to "Image search", "Transformerbased" to "Transformer-based" or "Transformer based". Input Citation: ``` {citation_text} ``` Title:""" def extract_paper_title_from_citation(citation_text: str): instruction = extract_title_prompt_template.format(citation_text=citation_text) response_dict = call_gemini_with_text_prompt( prompt=instruction, model_name="gemini-3-flash-preview", check_parsed_response_not_none=False, ) return response_dict["raw_response"].strip() - utils/gemini_utils.py (12393 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import json from typing import Dict, Any from google import genai from google.genai import types # type: ignore import time import os import mimetypes import base64 import threading import dotenv dot_file = os.path.join(os.path.dirname(__file__), "../.env") dotenv.load_dotenv(dot_file) _genai_client = None _genai_client_lock = threading.Lock() def get_genai_client() -> genai.Client: """Returns a shared Gemini client, creating it on first use. The client is created lazily so that Gemini credentials are only required when a Gemini model is actually called (e.g. OpenAI-only setups don't need them). Vertex AI takes precedence over GEMINI_API_KEY when both are set. """ global _genai_client if _genai_client is None: with _genai_client_lock: if _genai_client is None: vertex_ai_project = os.getenv("VERTEX_AI_PROJECT") vertex_ai_location = os.getenv("VERTEX_AI_LOCATION") gemini_api_key = os.getenv("GEMINI_API_KEY") if vertex_ai_project and vertex_ai_location: print( f"Using Vertex AI configuration. Project: {vertex_ai_project}, Location: {vertex_ai_location}" ) _genai_client = genai.Client( vertexai=True, project=vertex_ai_project, location=vertex_ai_location, ) elif gemini_api_key: print("Using Gemini API Key configuration.") _genai_client = genai.Client(api_key=gemini_api_key) else: raise ValueError( "Either VERTEX_AI_PROJECT/VERTEX_AI_LOCATION or GEMINI_API_KEY" " must be set to use Gemini models." ) return _genai_client def parse_gemini_json_results(response: str): if not response or not isinstance(response, str): return None json_pattern = r"```json(.*?)```" matches = re.findall(json_pattern, response, re.DOTALL) if not matches: json_pattern = r"(\{.*\}|\[.*\])" matches = re.findall(json_pattern, response, re.DOTALL) for json_string in matches: json_string = json_string.strip() try: parsed_json = json.loads(json_string) return parsed_json except json.JSONDecodeError: try: json_string_clean = re.sub(r"[\x00-\x1F\x7F]", "", json_string) json_string_clean = re.sub(r",\s*([\]}])", r"\1", json_string_clean) json_string_clean = re.sub( r'\\(?![\\"/bfnrtu])', r"\\\\", json_string_clean ) parsed_json = json.loads(json_string_clean) return parsed_json except json.JSONDecodeError: continue try: start_idx = response.find("{") end_idx = response.rfind("}") if start_idx != -1 and end_idx != -1 and end_idx > start_idx: json_string = response[start_idx : end_idx + 1] json_string_clean = re.sub(r"[\x00-\x1F\x7F]", "", json_string) json_string_clean = re.sub(r",\s*([\]}])", r"\1", json_string_clean) json_string_clean = re.sub( r'\\(?![\\"/bfnrtu])', r"\\\\", json_string_clean ) return json.loads(json_string_clean) except Exception: pass return None # No valid JSON found def parse_gemini_latex_results(response: str): match = re.search(r"```latex\s*(.*?)\s*```", response.strip(), re.DOTALL) if match: latex_string = match.group(1) else: latex_string = response return latex_string def parse_gemini_json_latex_response(response_text: str) -> Dict[str, Any]: """ Extracts the JSON worklog and the LaTeX code from the LLM response. Expected format: ... arbitrary text ... ```json { ... } ``` ... arbitrary text ... ```latex ... ``` """ result = {"json": None, "latex": None} json_match = re.search(r"```json\s*(.*?)\s*```", response_text, re.DOTALL) if json_match: try: result["json"] = json.loads(json_match.group(1)) except json.JSONDecodeError: print(" >> Error: Failed to parse JSON worklog from response.") result["json"] = {"error": "Invalid JSON format returned"} latex_match = re.search(r"```latex\s*(.*?)\s*```", response_text, re.DOTALL) if latex_match: result["latex"] = latex_match.group(1) else: print(" >> Warning: No ```latex``` block found.") return json.dumps(result) def call_gemini_with_contents( contents: types.Content, model_name: str, max_retries: int = 5, base_interval_sec: int = 5, result_parsing_func: callable = parse_gemini_json_results, generation_configs: dict = {}, check_parsed_response_not_none: bool = True, ): """ Calls the Gemini API (text-only) with retry logic and exponential backoff. """ raw_response = None parsed_response = None genai_client = get_genai_client() for attempt in range(max_retries): try: response = genai_client.models.generate_content( model=model_name, contents=contents, config=types.GenerateContentConfig(**generation_configs), ) if not response or not hasattr(response, "text") or response.text is None: raise ValueError("Incomplete response from Gemini API or empty text") raw_response = response.text parsed_response = result_parsing_func(raw_response) if check_parsed_response_not_none: assert ( parsed_response is not None ), f"Parsed response is None, Here is the raw response:\n{raw_response}" return { "raw_response": raw_response, "parsed_response": parsed_response, } except Exception as e: error_msg = str(e) print( f"Warning: Gemini API call failed on attempt {attempt + 1} of {max_retries}. Error: {error_msg}", ) if ( "maximum number of tokens allowed" in error_msg.lower() or "400 bad request" in error_msg.lower() or "code: 400" in error_msg.lower() or "400 invalid_argument" in error_msg.lower() ): print("Detected input token count exceed. Not retrying.") raise e if attempt + 1 == max_retries: print(f"Error: All {max_retries} retry attempts failed.") raise e delay = base_interval_sec * (2 ** (attempt - 1)) if attempt > 0 else 0 print(f"Retrying in {delay} seconds...") if delay > 0: time.sleep(delay) print("Error: Exited retry loop unexpectedly.") return {"raw_response": raw_response, "parsed_response": parsed_response} def call_gemini_with_text_prompt( prompt: str, model_name: str, result_parsing_func: callable = parse_gemini_json_results, generation_configs: dict = {}, check_parsed_response_not_none: bool = True, max_retries: int = 5, base_interval_sec: int = 5, ): contents = [types.Part(text=prompt)] return call_gemini_with_contents( contents=contents, model_name=model_name, result_parsing_func=result_parsing_func, generation_configs=generation_configs, check_parsed_response_not_none=check_parsed_response_not_none, max_retries=max_retries, base_interval_sec=base_interval_sec, ) def call_gemini_with_images( prompt: str, images: list[str], model_name: str, result_parsing_func: callable = parse_gemini_json_results, generation_configs: dict = {}, check_parsed_response_not_none: bool = True, max_retries: int = 5, base_interval_sec: int = 5, ): llm_contents = [ types.Part.from_text(text=prompt), ] for image_path in images: with open(image_path, "rb") as f: img_bytes = f.read() llm_contents.append( types.Part.from_bytes(data=img_bytes, mime_type="image/png") ) response_dict = call_gemini_with_contents( contents=llm_contents, model_name=model_name, generation_configs=generation_configs, result_parsing_func=result_parsing_func, check_parsed_response_not_none=check_parsed_response_not_none, max_retries=max_retries, base_interval_sec=base_interval_sec, ) return response_dict def generate_image_with_gemini( model_name: str, prompt: str, aspect_ratio: str = "16:9", generation_configs: dict = None, max_retries: int = 5, base_interval_sec: int = 5, save_path: str = None, ) -> str: """ Directly calls Gemini for image generation and returns base64. """ if generation_configs is None: generation_configs = {} genai_client = get_genai_client() for attempt in range(max_retries): try: config_args = { "temperature": 0.7, "candidate_count": 1, "response_modalities": ["IMAGE"], "image_config": types.ImageConfig( aspect_ratio=aspect_ratio, image_size="1k" ), } config_args.update(generation_configs) response = genai_client.models.generate_content( model=model_name, contents=[types.Part.from_text(text=prompt)], config=types.GenerateContentConfig(**config_args), ) if hasattr(response, "candidates") and response.candidates: if ( len(response.candidates) > 0 and response.candidates[0].content and response.candidates[0].content.parts ): for part in response.candidates[0].content.parts: if part.inline_data: img_base64 = base64.b64encode(part.inline_data.data).decode( "utf-8" ) if save_path: try: os.makedirs( os.path.dirname(save_path), exist_ok=True ) with open(save_path, "wb") as fh: fh.write(base64.b64decode(img_base64)) print( f"Image for prompt '{prompt}' saved to {save_path}" ) except Exception as e: print( f"Warning: Failed to save image to {save_path}: {e}" ) return img_base64 except Exception as e: print(f"Warning: Gemini API image generation failed {e}.") if attempt + 1 == max_retries: print( f"Error: All {max_retries} retry attempts failed for image generation." ) break delay = base_interval_sec * (2 ** (attempt - 1)) if attempt > 0 else 0 print(f"Retrying in {delay} seconds...") if delay > 0: time.sleep(delay) return None - utils/llm_backend_utils.py (7243 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from typing import Optional, Callable, Dict, Any from google.genai import types from utils.gemini_utils import ( call_gemini_with_text_prompt, parse_gemini_json_results, call_gemini_with_contents, call_gemini_with_images, ) from utils.openai_utils import ( call_openai_models_with_text_prompt, call_openai_models_with_content, parse_openai_json_results, get_openai_image_part_from_path, ) from utils.pdf_utils import load_paper def identity_parse(response: str): return response def get_llm_parser(model_name: str, return_json: bool = True) -> Callable: if not return_json: return identity_parse if ( "gpt" in model_name.lower() or "o1" in model_name.lower() or "o3" in model_name.lower() ): return parse_openai_json_results else: return parse_gemini_json_results def call_llm_with_text_prompt( prompt: str, model_name: str, generation_configs: Optional[Dict[str, Any]] = None, check_parsed_response_not_none: bool = True, return_json: bool = True, result_parsing_func: Optional[Callable] = None, ) -> Dict[str, Any]: """ Unified function to call either Gemini or OpenAI models based on the model_name. """ parser = ( result_parsing_func if result_parsing_func is not None else get_llm_parser(model_name, return_json) ) if generation_configs is None: generation_configs = {} if ( "gpt" in model_name.lower() or "o1" in model_name.lower() or "o3" in model_name.lower() ): system_prompt = None configs_for_openai = dict(generation_configs) if generation_configs else {} if "system_instruction" in configs_for_openai: system_prompt = configs_for_openai.pop("system_instruction") if "temperature" in configs_for_openai: configs_for_openai.pop("temperature") response_dict = call_openai_models_with_text_prompt( prompt=prompt, model_name=model_name, generation_configs=configs_for_openai, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, system_prompt=system_prompt, ) return response_dict else: response_dict = call_gemini_with_text_prompt( prompt=prompt, model_name=model_name, generation_configs=generation_configs, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, ) return response_dict def call_llm_with_images( prompt: str, images: list[str], model_name: str, generation_configs: Optional[Dict[str, Any]] = None, check_parsed_response_not_none: bool = True, return_json: bool = True, result_parsing_func: Optional[Callable] = None, ) -> Dict[str, Any]: """ Unified function to call either Gemini or OpenAI models with images based on the model_name. """ parser = ( result_parsing_func if result_parsing_func is not None else get_llm_parser(model_name, return_json) ) if generation_configs is None: generation_configs = {} if ( "gpt" in model_name.lower() or "o1" in model_name.lower() or "o3" in model_name.lower() ): system_prompt = None # Copy to avoid side-effects on caller's dictionary configs_for_openai = dict(generation_configs) if generation_configs else {} if "system_instruction" in configs_for_openai: system_prompt = configs_for_openai.pop("system_instruction") if "temperature" in configs_for_openai: configs_for_openai.pop("temperature") content = [{"type": "text", "text": prompt}] for image_path in images: content.extend(get_openai_image_part_from_path(image_path)) response_dict = call_openai_models_with_content( content=content, model_name=model_name, generation_configs=configs_for_openai, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, system_prompt=system_prompt, ) return response_dict else: response_dict = call_gemini_with_images( prompt=prompt, images=images, model_name=model_name, generation_configs=generation_configs, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, ) return response_dict def call_llm_with_pdf( pdf_path: str, prompt: str, model_name: str, system_instruction: Optional[str] = None, temperature: float = 0.7, check_parsed_response_not_none: bool = True, return_json: bool = True, result_parsing_func: Optional[Callable] = None, ) -> Dict[str, Any]: """ Unified function to evaluate a PDF. Passes raw PDF bytes to Gemini, and extracted text to OpenAI. """ parser = ( result_parsing_func if result_parsing_func is not None else get_llm_parser(model_name, return_json) ) if ( "gpt" in model_name.lower() or "o1" in model_name.lower() or "o3" in model_name.lower() ): paper_text = load_paper(pdf_path, min_size=100) full_prompt = f"Paper Content:\n{paper_text}\n\nTask:\n{prompt}" generation_configs = {} response_dict = call_openai_models_with_text_prompt( prompt=full_prompt, model_name=model_name, generation_configs=generation_configs, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, system_prompt=system_instruction, ) return response_dict else: with open(pdf_path, "rb") as f: pdf_bytes = f.read() contents = [ types.Part.from_bytes(data=pdf_bytes, mime_type="application/pdf"), types.Part.from_text(text=prompt), ] generation_configs = {"temperature": temperature} if system_instruction is not None: generation_configs["system_instruction"] = system_instruction response_dict = call_gemini_with_contents( contents=contents, model_name=model_name, generation_configs=generation_configs, result_parsing_func=parser, check_parsed_response_not_none=check_parsed_response_not_none, ) return response_dict - utils/openai_utils.py (7077 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Any import re import json import time import os import sys import threading import cv2 # type: ignore import base64 from openai import OpenAI # type: ignore from dotenv import load_dotenv # type: ignore dot_file = os.path.join(os.path.dirname(__file__), "../.env") load_dotenv(dot_file) _openai_client = None _openai_client_lock = threading.Lock() def get_openai_client() -> OpenAI: """Returns a shared OpenAI client, creating it on first use. The client is created lazily so that OPENAI_API_KEY is only required when an OpenAI model is actually called (e.g. Gemini-only setups don't need it). """ global _openai_client if _openai_client is None: with _openai_client_lock: if _openai_client is None: openai_api_key = os.environ.get("OPENAI_API_KEY") if not openai_api_key: raise ValueError("OPENAI_API_KEY must be set to use OpenAI models.") _openai_client = OpenAI(api_key=openai_api_key) return _openai_client def parse_openai_json_results(response: str): if not response or not isinstance(response, str): return None json_pattern = r"```json(.*?)```" matches = re.findall(json_pattern, response, re.DOTALL) if not matches: json_pattern = r"(\{.*\}|\[.*\])" matches = re.findall(json_pattern, response, re.DOTALL) for json_string in matches: json_string = json_string.strip() try: parsed_json = json.loads(json_string) return parsed_json except json.JSONDecodeError: try: json_string_clean = re.sub(r"[\x00-\x1F\x7F]", "", json_string) json_string_clean = re.sub(r",\s*([\]}])", r"\1", json_string_clean) json_string_clean = re.sub( r'\\(?![\\"/bfnrtu])', r"\\\\", json_string_clean ) parsed_json = json.loads(json_string_clean) return parsed_json except json.JSONDecodeError: continue try: start_idx = response.find("{") end_idx = response.rfind("}") if start_idx != -1 and end_idx != -1 and end_idx > start_idx: json_string = response[start_idx : end_idx + 1] json_string_clean = re.sub(r"[\x00-\x1F\x7F]", "", json_string) json_string_clean = re.sub(r",\s*([\]}])", r"\1", json_string_clean) json_string_clean = re.sub( r'\\(?![\\"/bfnrtu])', r"\\\\", json_string_clean ) return json.loads(json_string_clean) except Exception: pass try: start_idx = response.find("[") end_idx = response.rfind("]") if start_idx != -1 and end_idx != -1 and end_idx > start_idx: json_string = response[start_idx : end_idx + 1] json_string_clean = re.sub(r"[\x00-\x1F\x7F]", "", json_string) json_string_clean = re.sub(r",\s*([\]}])", r"\1", json_string_clean) json_string_clean = re.sub( r'\\(?![\\"/bfnrtu])', r"\\\\", json_string_clean ) return json.loads(json_string_clean) except Exception: pass return None # No valid JSON found def call_openai_models_with_content( content: list[Any], model_name: str, max_retries: int = 5, base_interval_sec: int = 5, result_parsing_func: callable = parse_openai_json_results, generation_configs: dict = {}, check_parsed_response_not_none: bool = True, system_prompt: str = None, ): raw_response = None parsed_response = None openai_client = get_openai_client() for attempt in range(max_retries): try: messages = [] if system_prompt: role = "developer" if model_name.startswith(("o1", "o3")) else "system" messages.append({"role": role, "content": system_prompt}) messages.append({"role": "user", "content": content}) response = openai_client.chat.completions.create( model=model_name, messages=messages, **generation_configs, ) raw_response = response.choices[0].message.content parsed_response = result_parsing_func(raw_response) if check_parsed_response_not_none: assert ( parsed_response is not None ), f"Parsed response is None, Here is the raw response:\n{raw_response}" return { "raw_response": raw_response, "parsed_response": parsed_response, } except Exception as e: print( f"Warning: OpenAI API call (model={model_name}) failed on attempt {attempt + 1} of {max_retries}. Error: {e}", ) if attempt + 1 == max_retries: print(f"Error: All {max_retries} retry attempts failed.") raise e delay = base_interval_sec * (2 ** (attempt - 1)) if attempt > 0 else 0 print(f"Retrying in {delay} seconds...") if delay > 0: time.sleep(delay) print("Error: Exited retry loop unexpectedly.") return { "raw_response": raw_response, "parsed_response": parsed_response, } def call_openai_models_with_text_prompt( prompt: str, model_name: str, result_parsing_func: callable = parse_openai_json_results, generation_configs: dict = {}, check_parsed_response_not_none: bool = True, max_retries: int = 5, base_interval_sec: int = 5, system_prompt: str = None, ): return call_openai_models_with_content( content=[{"type": "text", "text": prompt}], model_name=model_name, result_parsing_func=result_parsing_func, generation_configs=generation_configs, check_parsed_response_not_none=check_parsed_response_not_none, max_retries=max_retries, base_interval_sec=base_interval_sec, system_prompt=system_prompt, ) def get_openai_image_part_from_path(image_path: str): with open(image_path, "rb") as image_file: base64_image = base64.b64encode(image_file.read()).decode("utf-8") image_content = [ { "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{base64_image}", }, } ] return image_content - utils/paper_banana_utils.py (34689 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import base64 import io import re import os import json import matplotlib.pyplot as plt import json_repair from typing import Dict, Any, List, Tuple from google.genai import types from utils.gemini_utils import ( call_gemini_with_text_prompt, call_gemini_with_contents, generate_image_with_gemini, ) cur_dir = os.path.dirname(os.path.realpath(__file__)) PB_DIR = os.path.join(cur_dir, "../../PaperBanana") # ========================================== # PROMPTS # ========================================== # --- RETRIEVER --- DIAGRAM_RETRIEVER_AGENT_SYSTEM_PROMPT = """ # Background & Goal We are building an **AI system to automatically generate method diagrams for academic papers**. Given a paper's methodology section and a figure caption, the system needs to create a high-quality illustrative diagram that visualizes the described method. To help the AI learn how to generate appropriate diagrams, we use a **few-shot learning approach**: we provide it with reference examples of similar diagrams. The AI will learn from these examples to understand what kind of diagram to create for the target. # Your Task **You are the Retrieval Agent.** Your job is to select the most relevant reference diagrams from a candidate pool that will serve as few-shot examples for the diagram generation model. You will receive: - **Target Input:** The methodology section and caption of the diagram we need to generate - **Candidate Pool:** ~200 existing diagrams (each with methodology and caption) You must select the **Top 10 candidates** that would be most helpful as examples for teaching the AI how to draw the target diagram. # Selection Logic (Topic + Intent) Your goal is to find examples that match the Target in both **Domain** and **Diagram Type**. **1. Match Research Topic (Use Methodology & Caption):** * What is the domain? (e.g., Agent & Reasoning, Vision & Perception, Generative & Learning, Science & Applications). * Select candidates that belong to the **same research domain**. * *Why?* Similar domains share similar terminology (e.g., "Actor-Critic" in RL). **2. Match Visual Intent (Use Caption & Keywords):** * What type of diagram is implied? (e.g., "Framework", "Pipeline", "Detailed Module", "Performance Chart"). * Select candidates with **similar visual structures**. * *Why?* A "Framework" diagram example is useless for drawing a "Performance Bar Chart", even if they are in the same domain. **Ranking Priority:** 1. **Best Match:** Same Topic AND Same Visual Intent (e.g., Target is "Agent Framework" -> Candidate is "Agent Framework", Target is "Dataset Construction Pipeline" -> Candidate is "Dataset Construction Pipeline"). 2. **Second Best:** Same Visual Intent (e.g., Target is "Agent Framework" -> Candidate is "Vision Framework"). *Structure is more important than Topic for drawing.* 3. **Avoid:** Different Visual Intent (e.g., Target is "Pipeline" -> Candidate is "Bar Chart"). # Output Format Provide your output strictly in the following JSON format, containing only the **exact IDs** of the Top 10 selected diagrams (use the exact IDs from the Candidate Pool, such as "ref_1", etc.): ```json { "top10_diagrams": ["ref_1", "ref_25", "ref_100"] } ``` """ PLOT_RETRIEVER_AGENT_SYSTEM_PROMPT = """ # Background & Goal We are building an **AI system to automatically generate statistical plots**. Given a plot's raw data and the visual intent, the system needs to create a high-quality visualization that effectively presents the data. To help the AI learn how to generate appropriate plots, we use a **few-shot learning approach**: we provide it with reference examples of similar plots. The AI will learn from these examples to understand what kind of plot to create for the target data. # Your Task **You are the Retrieval Agent.** Your job is to select the most relevant reference plots from a candidate pool that will serve as few-shot examples for the plot generation model. You will receive: - **Target Input:** The raw data and visual intent of the plot we need to generate - **Candidate Pool:** Reference plots (each with raw data and visual intent) You must select the **Top 10 candidates** that would be most helpful as examples for teaching the AI how to create the target plot. # Selection Logic (Data Type + Visual Intent) Your goal is to find examples that match the Target in both **Data Characteristics** and **Plot Type**. **1. Match Data Characteristics (Use Raw Data & Visual Intent):** * What type of data is it? (e.g., categorical vs numerical, single series vs multi-series, temporal vs comparative). * What are the data dimensions? (e.g., 1D, 2D, 3D). * Select candidates with **similar data structures and characteristics**. * *Why?* Different data types require different visualization approaches. **2. Match Visual Intent (Use Visual Intent):** * What type of plot is implied? (e.g., "bar chart", "scatter plot", "line chart", "pie chart", "heatmap", "radar chart"). * Select candidates with **similar plot types**. * *Why?* A "bar chart" example is more useful for generating another bar chart than a "scatter plot" example. **Ranking Priority:** 1. **Best Match:** Same Data Type AND Same Plot Type (e.g., Target is "multi-series line chart" -> Candidate is "multi-series line chart"). 2. **Second Best:** Same Plot Type with compatible data. 3. **Avoid:** Different Plot Type. # Output Format Provide your output strictly in the following JSON format, containing only the **exact Plot IDs** of the Top 10 selected plots (use the exact IDs from the Candidate Pool, such as "ref_0", etc.): ```json { "top10_plots": ["ref_0", "ref_25", "ref_100"] } ``` """ # --- PLANNER --- DIAGRAM_PLANNER_AGENT_SYSTEM_PROMPT = """ I am working on a task: given the 'Methodology' section of a paper, and the caption of the desired figure, automatically generate a corresponding illustrative diagram. I will input the text of the 'Methodology' section, the figure caption, and your output should be a detailed description of an illustrative figure that effectively represents the methods described in the text. To help you understand the task better, and grasp the principles for generating such figures, I will also provide you with several examples. You should learn from these examples to provide your figure description. ** IMPORTANT: ** Your description should be as detailed as possible. Semantically, clearly describe each element and their connections. Formally, include various details such as background style (typically pure white or very light pastel), colors, line thickness, icon styles, etc. Remember: vague or unclear specifications will only make the generated figure worse, not better. """ PLOT_PLANNER_AGENT_SYSTEM_PROMPT = """ I am working on a task: given the raw data (typically in tabular or json format) and a visual intent of the desired plot, automatically generate a corresponding statistical plot that are both accurate and aesthetically pleasing. I will input the raw data and the plot visual intent, and your output should be a detailed description of an illustrative plot that effectively represents the data. Note that your description should include all the raw data points to be plotted. To help you understand the task better, and grasp the principles for generating such plots, I will also provide you with several examples. You should learn from these examples to provide your plot description. ** IMPORTANT: ** Your description should be as detailed as possible. For content, explain the precise mapping of variables to visual channels (x, y, hue) and explicitly enumerate every raw data point's coordinate to be drawn to ensure accuracy. For presentation, specify the exact aesthetic parameters, including specific HEX color codes, font sizes for all labels, line widths, marker dimensions, legend placement, and grid styles. You should learn from the examples' content presentation and aesthetic design (e.g., color schemes). """ # --- STYLIST --- DIAGRAM_STYLIST_AGENT_SYSTEM_PROMPT = """ ## ROLE You are a Lead Visual Designer for top-tier AI conferences (e.g., NeurIPS 2025). ## TASK Our goal is to generate high-quality, publication-ready diagrams, given the methodology section and the caption of the desired diagram. The diagram should illustrate the logic of the methodology section, while adhering to the scope defined by the caption. Before you, a planner agent has already generated a preliminary description of the target diagram. However, this description may lack specific aesthetic details, such as element shapes, color palettes, and background styling. Your task is to refine and enrich this description based on the provided [NeurIPS 2025 Style Guidelines] to ensure the final generated image is a high-quality, publication-ready diagram that adheres to the NeurIPS 2025 aesthetic standards where appropriate. ## INPUT DATA - **Detailed Description**: [The preliminary description of the figure] - **Style Guidelines**: [NeurIPS 2025 Style Guidelines] - **Methodology Section**: [Contextual content from the methodology section] - **Diagram Caption**: [Target diagram caption] Note that you should primary focus on the detailed description and style guidelines. The methodology section and diagram caption are provided for context only, there's no need to regenerate a description from scratch, solely based on them, while ignoring the detailed description we already have. **Crucial Instructions:** 1. **Preserve Semantic Content:** Do NOT alter the semantic content, logic, or structure of the diagram. Your job is purely aesthetic refinement, not content editing. However, if you find some phrases or descriptions too verbose, you may simplify them appropriately while referencing the original methodology section to ensure semantic accuracy. 2. **Preserve High-Quality Aesthetics and Intervene Only When Necessary:** First, evaluate the aesthetic quality implied by the input description. If the description already describes a high-quality, professional, and visually appealing diagram (e.g., nice 3D icons, rich textures, good color harmony), **PRESERVE IT**. Only apply strict Style Guide adjustments if the current description lacks detail, looks outdated, or is visually cluttered. Your goal is specific refinement, not blind standardization. 3. **Respect Diversity:** Different domains have different styles. If the input describes a specific style (e.g., illustrative for agents) that works well, keep it. 4. **Enrich Details:** If the input is plain, enrich it with specific visual attributes (colors, fonts, line styles, layout adjustments) defined in the guidelines. 5. **Handle Icons with Care:** Be cautious when modifying icons as they may carry specific semantic meanings. Some icons have conventional technical meanings (e.g., snowflake = frozen/non-trainable, flame = trainable) - when encountering such icons, reference the original methodology section to verify their intent before making changes. However, purely decorative or symbolic icons can be freely enhanced and beautified. For examples, agent papers often use cute 2D robot avatars to represent agents. ## OUTPUT Output ONLY the final polished Detailed Description. Do not include any conversational text or explanations. """ PLOT_STYLIST_AGENT_SYSTEM_PROMPT = """ ## ROLE You are a Lead Visual Designer for top-tier AI conferences (e.g., NeurIPS 2025). ## TASK You are provided with a preliminary description of a statistical plot to be generated. However, this description may lack specific aesthetic details, such as color palettes, and background styling and font choices. Your task is to refine and enrich this description based on the provided [NeurIPS 2025 Style Guidelines] to ensure the final generated image is a high-quality, publication-ready plot that strictly adheres to the NeurIPS 2025 aesthetic standards. **Crucial Instructions:** 1. **Enrich Details:** Focus on specifying visual attributes (colors, fonts, line styles, layout adjustments) defined in the guidelines. 2. **Preserve Content:** Do NOT alter the semantic content, logic, or quantitative results of the plot. Your job is purely aesthetic refinement, not content editing. 3. **Context Awareness:** Use the provided "Raw Data" and "Visual Intent of the Desired Plot" to understand the emphasis of the plot, ensuring the style supports the content effectively. ## INPUT DATA - **Detailed Description**: [The preliminary description of the plot] - **Style Guidelines**: [NeurIPS 2025 Style Guidelines] - **Raw Data**: [The raw data to be visualized] - **Visual Intent of the Desired Plot**: [Visual intent of the desired plot] ## OUTPUT Output ONLY the final polished Detailed Description. Do not include any conversational text or explanations. """ # --- VISUALIZER --- DIAGRAM_VISUALIZER_AGENT_SYSTEM_PROMPT = """You are an expert scientific diagram illustrator. Generate high-quality scientific diagrams based on user requests.""" PLOT_VISUALIZER_AGENT_SYSTEM_PROMPT = """You are an expert statistical plot illustrator. Write code to generate high-quality statistical plots based on user requests.""" # --- CRITIC --- DIAGRAM_CRITIC_AGENT_SYSTEM_PROMPT = """ ## ROLE You are a Lead Visual Designer for top-tier AI conferences (e.g., NeurIPS 2025). ## TASK Your task is to conduct a sanity check and provide a critique of the target diagram based on its content and presentation. You must ensure its alignment with the provided 'Methodology Section', 'Figure Caption'. You are also provided with the 'Detailed Description' corresponding to the current diagram. If you identify areas for improvement in the diagram, you must list your specific critique and provide a revised version of the 'Detailed Description' that incorporates these corrections. ## CRITIQUE & REVISION RULES 1. Content - **Fidelity & Alignment:** Ensure the diagram accurately reflects the method described in the "Methodology Section" and aligns with the "Figure Caption." Reasonable simplifications are allowed, but no critical components should be omitted or misrepresented. Also, the diagram should not contain any hallucinated content. Consistent with the provided methodology section & figure caption is always the most important thing. - **Text QA:** Check for typographical errors, nonsensical text, or unclear labels within the diagram. Suggest specific corrections. - * - utils/pdf_utils.py (9854 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import subprocess import json import shutil import rich import traceback import subprocess import os.path as osp import pymupdf4llm # type: ignore import pymupdf # type: ignore import numpy as np import cv2 from pypdf import PdfReader # type: ignore def load_paper(pdf_path, num_pages=None, min_size=100): try: doc = pymupdf.open(pdf_path) if num_pages: doc = doc[:num_pages] text = "" for page in doc: text += page.get_text() if len(text) < min_size: raise Exception("Text too short") except Exception as e: print(f"Error with pymupdf, falling back to pypdf: {e}") reader = PdfReader(pdf_path) if num_pages is None: pages = reader.pages else: pages = reader.pages[:num_pages] text = "".join(page.extract_text() for page in pages) if len(text) < min_size: raise Exception("Text too short") return text def has_blacklist_words(text: str): blacklists = ["corresponding author"] for word in blacklists: if word in text.lower(): return True return False def parse_sections(json_data, output_folder=""): if not output_folder: output_folder = os.path.dirname(json_data) output_path = os.path.join(output_folder, "paper_sections.json") section_results = [] if "abstractText" in json_data: abstract_data = json_data["abstractText"]["text"] if abstract_data.startswith("Abstract"): abstract_data = abstract_data[len("Abstract") :] section_results.append({"title": "Abstract", "content": abstract_data}) section_data = json_data["sections"] for section in section_data: title = section["title"]["text"] if "title" in section else None content = "" for para in section["paragraphs"]: if has_blacklist_words(para["text"]): continue content += para["text"] if title: section_results.append({"title": title, "content": content}) else: section_results[-1]["content"] += content with open(output_path, "w") as f: json.dump(section_results, f, indent=4) print(f"Dumped paper sections to {output_path}.") return section_results papaer_text_to_reference_text_prompt_template = """ You are a specialized academic data extraction engine. Your task is to extract the "References" or "Bibliography" section from the raw text of a research paper and return it in a specific single-line string format. ### INPUT DATA You will receive raw text extracted from a PDF. This text may contain: - Noise (headers, footers, page numbers). - Arbitrary line breaks (sentences broken across lines). - The body of the paper followed by the references. ### INSTRUCTIONS 1. **Locate:** Find the start of the "References" or "Bibliography" section. Ignore all text preceding this section. 2. **Extract:** Identify individual reference entries. These typically start with bracketed numbers (e.g., [1], [2]) or bare numbers (1., 2.). 3. **Clean:** - Merge multi-line citations into a single line. - Remove any page numbers or running headers that interrupt a citation. 4. **Format:** Output the citations as a single continuous string. Ensure every citation starts with its bracketed ID (e.g., `[1]`). If the source text uses `1.` format, convert it to `[1]`. ### OUTPUT FORMAT Your output must be a single string containing ONLY the references, formatted exactly as follows: "[1] First citation text [2] Second citation text [3] Third citation text [4] ... " ### CONSTRAINTS - Do NOT output JSON, Markdown lists, or XML. - Do NOT output the word "References" or "Bibliography" at the start. - Do NOT output any conversational text (e.g., "Here are the references"). - Do NOT change the content/wording of the citation titles or authors, only clean up the whitespace. Here is the paper text: [PAPER CONTENT] {paper_text} [END PAPER CONTENT] """ def extract_reference_from_pdf_text( paper_text: str, model_name: str = "gemini-3.1-pro-preview" ) -> str: from utils.llm_backend_utils import call_llm_with_text_prompt instructions = papaer_text_to_reference_text_prompt_template.format( paper_text=paper_text ) response_dict = call_llm_with_text_prompt( prompt=instructions, model_name=model_name, check_parsed_response_not_none=False, return_json=False, ) return response_dict["raw_response"].strip() def get_paper_references_from_pdf( pdf_path: str, model_name: str = "gemini-3.1-pro-preview" ): try: paper_text = load_paper(pdf_path) reference_text = extract_reference_from_pdf_text( paper_text, model_name=model_name ) if reference_text: return reference_text else: print( f"Method 2: Failed to extract references from {pdf_path}.\nReturn None for refernces." ) except Exception as e: print( f"Method 2: Failed to extract references from {pdf_path} due to {str(e)}.\nReturn None for refernces." ) return None def compile_latex(cwd, pdf_file, texfile_name="template", timeout=30): import os, shutil, subprocess, traceback, rich source_pdf = os.path.join(cwd, f"{texfile_name}.pdf") if os.path.exists(source_pdf): os.remove(source_pdf) env = dict(os.environ) for var in ["TEXINPUTS", "TEXMFHOME", "TEXMF", "TEXMFVAR", "TEXMFCONFIG"]: env.pop(var, None) commands = [ ["pdflatex", "-interaction=nonstopmode", f"{texfile_name}.tex"], ["bibtex", texfile_name], ["pdflatex", "-interaction=nonstopmode", f"{texfile_name}.tex"], ["pdflatex", "-interaction=nonstopmode", f"{texfile_name}.tex"], ] execution_logs = [] for command in commands: cmd_str = " ".join(command) try: result = subprocess.run( command, cwd=cwd, capture_output=True, # Captures stdout/stderr silently text=True, timeout=timeout, env=env, ) execution_logs.append( { "cmd": cmd_str, "returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr, } ) except subprocess.TimeoutExpired: execution_logs.append( {"cmd": cmd_str, "error": f"TIMEOUT after {timeout}s"} ) except Exception as e: execution_logs.append({"cmd": cmd_str, "error": str(e)}) has_timeout = any( "error" in log and "TIMEOUT" in log["error"] for log in execution_logs ) if os.path.exists(source_pdf) and not has_timeout: try: shutil.move(source_pdf, pdf_file) except Exception: rich.print(f"[bold red]Error moving generated PDF to {pdf_file}[/bold red]") print(traceback.format_exc()) else: rich.print( f"\n[bold red]❌ PDF Generation Failed (No PDF file produced)[/bold red]" ) for log in execution_logs: if "error" in log: header_color = "red" elif log["returncode"] == 0: header_color = "green" else: header_color = "yellow" # Warning/Error rich.print( f"\n[bold {header_color}]---> Ran: {log['cmd']}[/bold {header_color}]" ) if "error" in log: rich.print(f"[bold red]SYSTEM ERROR: {log['error']}[/bold red]") else: if log["stdout"].strip(): rich.print(f"[dim]{log['stdout'].strip()}[/dim]") if log["stderr"].strip(): rich.print(f"[red]{log['stderr'].strip()}[/red]") def pdf_to_grid_images(pdf_path, output_dir, scale=2.0): """Converts a PDF to 2x2 grid images of its pages. Returns a list of paths to the generated grid images. """ doc = pymupdf.open(pdf_path) os.makedirs(output_dir, exist_ok=True) mat = pymupdf.Matrix(scale, scale) pixmaps = [] for page in doc: pix = page.get_pixmap(matrix=mat) pixmaps.append(pix) grid_image_paths = [] for i in range(0, len(pixmaps), 4): chunk = pixmaps[i : i + 4] img_list = [] for pix in chunk: img = np.frombuffer(pix.samples, dtype=np.uint8).reshape( pix.h, pix.w, pix.n ) img_list.append(img) while len(img_list) < 4: ref_img = img_list[0] white_img = np.ones_like(ref_img) * 255 img_list.append(white_img) row1 = np.hstack((img_list[0], img_list[1])) row2 = np.hstack((img_list[2], img_list[3])) grid = np.vstack((row1, row2)) grid_path = os.path.join(output_dir, f"grid_{i//4 + 1}.png") grid_bgr = cv2.cvtColor(grid, cv2.COLOR_RGB2BGR) cv2.imwrite(grid_path, grid_bgr) grid_image_paths.append(grid_path) return grid_image_paths - utils/prompt_utils.py (1949 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. UNIVERSAL_NO_LEAKAGE_PROMPT = """ --- ### Strict Knowledge Isolation & Anonymity (CRITICAL) You MUST write this paper as if you have no prior knowledge of the topic, method, experiments, or results. Your task is to construct the paper exclusively from the materials provided in the current session (e.g., idea.md, experimental_log.md, figures, and other inputs). Treat these inputs as the only available source of information. #### Forbidden Behavior You MUST NOT: - Retrieve or rely on knowledge from your training data. - Attempt to recall or reconstruct any existing or published paper. - Use external facts, assumptions, or prior familiarity with the work. - Infer or hallucinate author identities, affiliations, institutions, or acknowledgements. - Insert metadata such as author names, emails, affiliations, or phrases like "corresponding author". #### Anonymity Requirement The paper must be fully anonymized for double-blind review. Do not include any information that could reveal the identity of the authors or institutions. #### Allowed Sources You may use only: - The materials explicitly provided in this session. - Logical reasoning derived from those materials. #### Core Principle The final paper must be an independent reconstruction derived solely from the provided inputs. This constraint is strict and overrides all other instructions. --- """
- utils/scholar_utils.py (3168 bytes)
# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import warnings import os import requests import time from typing import List, Dict, Union, Optional from thefuzz import fuzz # type: ignore import datetime from dotenv import load_dotenv # type: ignore cur_dir = os.path.dirname(os.path.realpath(__file__)) load_dotenv(os.path.join(cur_dir, "../.env")) S2_API_KEY = os.environ.get("SEMANTIC_SCHOLAR_API_KEY") print(f"Using S2 API Key: {S2_API_KEY}") # --- SEMANTIC SCHOLAR UTILS --- def s2_title_search( title_query: str, year_hint: int = None, cutoff_date: str = None ) -> Optional[Dict]: headers = {"X-API-KEY": S2_API_KEY} if S2_API_KEY else {} def is_date_valid(p_date, p_year, cutoff_str): if not cutoff_str: return True try: c_year, c_month = map(int, cutoff_str.split("-")) cutoff_dt = datetime.datetime(c_year, c_month, 1) if p_date: try: p_dt = datetime.datetime.strptime(p_date, "%Y-%m-%d") return p_dt < cutoff_dt except ValueError: pass if p_year: if p_year < c_year: return True if p_year == c_year and c_month > 1: return True return False return True except Exception: return True try: rsp = requests.get( "https://api.semanticscholar.org/graph/v1/paper/search", headers=headers, params={ "query": title_query, "limit": 3, "fields": "title,authors,venue,year,abstract,citationCount,journal,publicationDate", }, timeout=5, ) if rsp.status_code != 200: return None results = rsp.json().get("data", []) if not results: return None best_match = None highest_ratio = 0 for r in results: if not r.get("title"): continue if not is_date_valid(r.get("publicationDate"), r.get("year"), cutoff_date): continue ratio = fuzz.ratio(title_query.lower(), r["title"].lower()) if year_hint and r.get("year") == year_hint: ratio += 10 if ratio > highest_ratio: highest_ratio = ratio best_match = r if highest_ratio > 70: return best_match return None except Exception as e: print(f" ⚠️ S2 Error for '{title_query}': {e}") return None