From 409a4acc57255e96df54cb70d01180ec9c8b7305 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 10:33:39 -0600 Subject: [PATCH 1/8] import claude conversations --- src/basic_memory/cli/app.py | 4 + .../cli/commands/import_claude.py | 204 ++++++++++++++++++ .../cli/commands/import_memory_json.py | 4 +- src/basic_memory/cli/main.py | 2 +- 4 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 src/basic_memory/cli/commands/import_claude.py diff --git a/src/basic_memory/cli/app.py b/src/basic_memory/cli/app.py index 43437328c..b1dc909ca 100644 --- a/src/basic_memory/cli/app.py +++ b/src/basic_memory/cli/app.py @@ -11,3 +11,7 @@ asyncio.run(db.run_migrations(config)) app = typer.Typer() + + +import_app = typer.Typer() +app.add_typer(import_app, name="import") \ No newline at end of file diff --git a/src/basic_memory/cli/commands/import_claude.py b/src/basic_memory/cli/commands/import_claude.py new file mode 100644 index 000000000..446ea0491 --- /dev/null +++ b/src/basic_memory/cli/commands/import_claude.py @@ -0,0 +1,204 @@ +"""Import command for basic-memory CLI to import chat data from conversations2.json format.""" + +import asyncio +import json +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, List, Annotated + +import typer +from loguru import logger +from rich.console import Console +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn + +from basic_memory.cli.app import app, import_app +from basic_memory.config import config +from basic_memory.markdown import EntityParser, MarkdownProcessor +from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter + +console = Console() + + +def clean_filename(text: str) -> str: + """Convert text to safe filename.""" + # Remove invalid characters and convert spaces + clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-") + return clean + + +def format_timestamp(ts: str) -> str: + """Format ISO timestamp for display.""" + dt = datetime.fromisoformat(ts.replace("Z", "+00:00")) + return dt.strftime("%Y-%m-%d %H:%M:%S") + + +def format_chat_markdown( + name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str +) -> str: + """Format chat as clean markdown.""" + + # Start with frontmatter and title + lines = [ + f"# {name}\n", + ] + + # Add messages + for msg in messages: + # Format timestamp + ts = format_timestamp(msg["created_at"]) + + # Add message header + lines.append(f"### {msg['sender'].title()} ({ts})") + + # Handle message content + content = msg.get("text", "") + if msg.get("content"): + content = " ".join(c.get("text", "") for c in msg["content"]) + lines.append(content) + + # Handle attachments + attachments = msg.get("attachments", []) + for attachment in attachments: + if "file_name" in attachment: + lines.append(f"\n**Attachment: {attachment['file_name']}**") + if "extracted_content" in attachment: + lines.append("```") + lines.append(attachment["extracted_content"]) + lines.append("```") + + # Add spacing between messages + lines.append("") + + return "\n".join(lines) + + +def format_chat_content( + name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str +) -> EntityMarkdown: + """Convert chat messages to Basic Memory entity format.""" + + # Generate permalink + date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d") + clean_title = clean_filename(name) + permalink = f"conversations/{date_prefix}-{clean_title}" + + # Format content + content = format_chat_markdown( + name=name, + messages=messages, + created_at=created_at, + modified_at=modified_at, + permalink=permalink, + ) + + # Create entity + entity = EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={ + "type": "conversation", + "title": name, + "created": created_at, + "modified": modified_at, + "permalink": permalink, + } + ), + content=content, + ) + + return entity + + +async def process_chat_json( + json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor +) -> Dict[str, int]: + """Import chat data from conversations2.json format.""" + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + console=console, + ) as progress: + read_task = progress.add_task("Reading chat data...", total=None) + + # Read chat data - handle array of arrays format + data = json.loads(json_path.read_text()) + conversations = [chat for chat in data] + progress.update(read_task, total=len(conversations)) + + # Process each conversation + messages_imported = 0 + chats_imported = 0 + + for chat in conversations: + # Convert to entity + entity = format_chat_content( + name=chat["name"], + messages=chat["chat_messages"], + created_at=chat["created_at"], + modified_at=chat["updated_at"], + ) + + # Write file + file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md" + await markdown_processor.write_file(file_path, entity) + + chats_imported += 1 + messages_imported += len(chat["chat_messages"]) + progress.update(read_task, advance=1) + + return {"conversations": chats_imported, "messages": messages_imported} + + +async def get_markdown_processor() -> MarkdownProcessor: + """Get MarkdownProcessor instance.""" + entity_parser = EntityParser(config.home) + return MarkdownProcessor(entity_parser) + + +@import_app.command(name="claude", help="Import chat conversations from claude.") +def import_conversations( + folder: str = Annotated[str, typer.Argument(default="conversations", help="The folder to place the files in.")], + json_path: Path = Annotated[Path, typer.Argument(..., help="Path to conversations.json file")], +): + """Import chat conversations from conversations2.json format. + + This command will: + 1. Read chat data and nested messages + 2. Create markdown files for each conversation + 3. Format content in clean, readable markdown + + After importing, run 'basic-memory sync' to index the new files. + """ + + if not json_path.exists(): + typer.echo(f"Error: File not found: {json_path}", err=True) + raise typer.Exit(1) + + try: + # Get markdown processor + markdown_processor = asyncio.run(get_markdown_processor()) + + # Process the file + base_path = config.home / folder + console.print(f"\nImporting chats from {json_path}...writing to {base_path}") + results = asyncio.run(process_chat_json(json_path, base_path, markdown_processor)) + + # Show results + console.print( + Panel( + f"[green]Import complete![/green]\n\n" + f"Imported {results['conversations']} conversations\n" + f"Containing {results['messages']} messages", + expand=False, + ) + ) + + console.print("\nRun 'basic-memory sync' to index the new files.") + + except Exception as e: + logger.exception("Import failed") + typer.echo(f"Error during import: {e}", err=True) + raise typer.Exit(1) diff --git a/src/basic_memory/cli/commands/import_memory_json.py b/src/basic_memory/cli/commands/import_memory_json.py index 6e8166fed..7f5029d41 100644 --- a/src/basic_memory/cli/commands/import_memory_json.py +++ b/src/basic_memory/cli/commands/import_memory_json.py @@ -11,7 +11,7 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn -from basic_memory.cli.app import app +from basic_memory.cli.app import app, import_app from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation @@ -98,7 +98,7 @@ async def get_markdown_processor() -> MarkdownProcessor: return MarkdownProcessor(entity_parser) -@app.command() +@import_app.command() def import_json( json_path: Path = typer.Argument(..., help="Path to memory.json file to import"), ): diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index 8512a4dc1..5457534a5 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -4,7 +4,7 @@ from basic_memory.utils import setup_logging # pragma: no cover # Register commands -from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp # noqa: F401 # pragma: no cover +from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp, import_claude # noqa: F401 # pragma: no cover # Set up logging when module is imported From f894bc1b87161263ba07fb55908c7c9d6247707d Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 11:06:41 -0600 Subject: [PATCH 2/8] refactor claude import args --- .../cli/commands/import_claude.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/basic_memory/cli/commands/import_claude.py b/src/basic_memory/cli/commands/import_claude.py index 446ea0491..3c4a969d0 100644 --- a/src/basic_memory/cli/commands/import_claude.py +++ b/src/basic_memory/cli/commands/import_claude.py @@ -4,7 +4,7 @@ import json from datetime import datetime from pathlib import Path -from typing import Dict, Any, List, Annotated +from typing import Dict, Any, List, Annotated, Optional import typer from loguru import logger @@ -109,7 +109,7 @@ def format_chat_content( return entity -async def process_chat_json( +async def process_conversations_json( json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor ) -> Dict[str, int]: """Import chat data from conversations2.json format.""" @@ -160,8 +160,8 @@ async def get_markdown_processor() -> MarkdownProcessor: @import_app.command(name="claude", help="Import chat conversations from claude.") def import_conversations( - folder: str = Annotated[str, typer.Argument(default="conversations", help="The folder to place the files in.")], - json_path: Path = Annotated[Path, typer.Argument(..., help="Path to conversations.json file")], + conversations_json: Annotated[Optional[Path], typer.Option(..., help="Path to conversations.json file")] = None, + conversations_folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", ): """Import chat conversations from conversations2.json format. @@ -173,28 +173,30 @@ def import_conversations( After importing, run 'basic-memory sync' to index the new files. """ - if not json_path.exists(): - typer.echo(f"Error: File not found: {json_path}", err=True) - raise typer.Exit(1) - try: - # Get markdown processor - markdown_processor = asyncio.run(get_markdown_processor()) - - # Process the file - base_path = config.home / folder - console.print(f"\nImporting chats from {json_path}...writing to {base_path}") - results = asyncio.run(process_chat_json(json_path, base_path, markdown_processor)) - - # Show results - console.print( - Panel( - f"[green]Import complete![/green]\n\n" - f"Imported {results['conversations']} conversations\n" - f"Containing {results['messages']} messages", - expand=False, + if conversations_json: + + if not conversations_json.exists(): + typer.echo(f"Error: File not found: {conversations_json}", err=True) + raise typer.Exit(1) + + # Get markdown processor + markdown_processor = asyncio.run(get_markdown_processor()) + + # Process the file + base_path = config.home / conversations_folder + console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}") + results = asyncio.run(process_conversations_json(conversations_json, base_path, markdown_processor)) + + # Show results + console.print( + Panel( + f"[green]Import complete![/green]\n\n" + f"Imported {results['conversations']} conversations\n" + f"Containing {results['messages']} messages", + expand=False, + ) ) - ) console.print("\nRun 'basic-memory sync' to index the new files.") From b2398b08fb1c2717f11cd9359376b2ffcce1c7d2 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 11:21:32 -0600 Subject: [PATCH 3/8] refactor claud conversations import args --- .../cli/commands/import_claude.py | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/basic_memory/cli/commands/import_claude.py b/src/basic_memory/cli/commands/import_claude.py index 3c4a969d0..a2795dd44 100644 --- a/src/basic_memory/cli/commands/import_claude.py +++ b/src/basic_memory/cli/commands/import_claude.py @@ -159,9 +159,9 @@ async def get_markdown_processor() -> MarkdownProcessor: @import_app.command(name="claude", help="Import chat conversations from claude.") -def import_conversations( - conversations_json: Annotated[Optional[Path], typer.Option(..., help="Path to conversations.json file")] = None, - conversations_folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", +def import_claude( + conversations_json: Annotated[Path, typer.Argument(..., help="Path to conversations.json file")] = "conversations.json", + folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", ): """Import chat conversations from conversations2.json format. @@ -174,29 +174,27 @@ def import_conversations( """ try: - if conversations_json: - - if not conversations_json.exists(): - typer.echo(f"Error: File not found: {conversations_json}", err=True) - raise typer.Exit(1) - - # Get markdown processor - markdown_processor = asyncio.run(get_markdown_processor()) + if not conversations_json.exists(): + typer.echo(f"Error: File not found: {conversations_json}", err=True) + raise typer.Exit(1) - # Process the file - base_path = config.home / conversations_folder - console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}") - results = asyncio.run(process_conversations_json(conversations_json, base_path, markdown_processor)) - - # Show results - console.print( - Panel( - f"[green]Import complete![/green]\n\n" - f"Imported {results['conversations']} conversations\n" - f"Containing {results['messages']} messages", - expand=False, - ) + # Get markdown processor + markdown_processor = asyncio.run(get_markdown_processor()) + + # Process the file + base_path = config.home / folder + console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}") + results = asyncio.run(process_conversations_json(conversations_json, base_path, markdown_processor)) + + # Show results + console.print( + Panel( + f"[green]Import complete![/green]\n\n" + f"Imported {results['conversations']} conversations\n" + f"Containing {results['messages']} messages", + expand=False, ) + ) console.print("\nRun 'basic-memory sync' to index the new files.") From 8b1066a2ee753d44fbcc86734e4577722b39530d Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 11:23:58 -0600 Subject: [PATCH 4/8] refactor claud conversations import args --- .../commands/{import_claude.py => import_claude_conversations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/basic_memory/cli/commands/{import_claude.py => import_claude_conversations.py} (100%) diff --git a/src/basic_memory/cli/commands/import_claude.py b/src/basic_memory/cli/commands/import_claude_conversations.py similarity index 100% rename from src/basic_memory/cli/commands/import_claude.py rename to src/basic_memory/cli/commands/import_claude_conversations.py From a3db217e04bbd9e0ac2d9213fcd818a47b892b59 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 11:25:11 -0600 Subject: [PATCH 5/8] separate import commands for claude --- src/basic_memory/cli/app.py | 7 +++++-- .../cli/commands/import_claude_conversations.py | 4 ++-- src/basic_memory/cli/main.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/basic_memory/cli/app.py b/src/basic_memory/cli/app.py index b1dc909ca..2fb907909 100644 --- a/src/basic_memory/cli/app.py +++ b/src/basic_memory/cli/app.py @@ -12,6 +12,9 @@ app = typer.Typer() - import_app = typer.Typer() -app.add_typer(import_app, name="import") \ No newline at end of file +app.add_typer(import_app, name="import") + + +claude_app = typer.Typer() +import_app.add_typer(claude_app, name="claude") \ No newline at end of file diff --git a/src/basic_memory/cli/commands/import_claude_conversations.py b/src/basic_memory/cli/commands/import_claude_conversations.py index a2795dd44..0af5d8986 100644 --- a/src/basic_memory/cli/commands/import_claude_conversations.py +++ b/src/basic_memory/cli/commands/import_claude_conversations.py @@ -12,7 +12,7 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn -from basic_memory.cli.app import app, import_app +from basic_memory.cli.app import app, import_app, claude_app from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter @@ -158,7 +158,7 @@ async def get_markdown_processor() -> MarkdownProcessor: return MarkdownProcessor(entity_parser) -@import_app.command(name="claude", help="Import chat conversations from claude.") +@claude_app.command(name="conversations", help="Import chat conversations from claude.") def import_claude( conversations_json: Annotated[Path, typer.Argument(..., help="Path to conversations.json file")] = "conversations.json", folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index 5457534a5..90bc784f9 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -4,7 +4,7 @@ from basic_memory.utils import setup_logging # pragma: no cover # Register commands -from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp, import_claude # noqa: F401 # pragma: no cover +from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp, import_claude_conversations # noqa: F401 # pragma: no cover # Set up logging when module is imported From 49e30e6a05c3657212da9ef2d80691928528ff84 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 11:54:35 -0600 Subject: [PATCH 6/8] update tests for import --- .../commands/import_claude_conversations.py | 9 +- .../cli/commands/import_claude_projects.py | 189 ++++++++++++++++++ .../cli/commands/import_memory_json.py | 2 +- src/basic_memory/cli/main.py | 10 +- tests/cli/test_import_memory_json.py | 10 +- 5 files changed, 209 insertions(+), 11 deletions(-) create mode 100644 src/basic_memory/cli/commands/import_claude_projects.py diff --git a/src/basic_memory/cli/commands/import_claude_conversations.py b/src/basic_memory/cli/commands/import_claude_conversations.py index 0af5d8986..40a7720e2 100644 --- a/src/basic_memory/cli/commands/import_claude_conversations.py +++ b/src/basic_memory/cli/commands/import_claude_conversations.py @@ -74,14 +74,14 @@ def format_chat_markdown( def format_chat_content( - name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str + base_path: str, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str ) -> EntityMarkdown: """Convert chat messages to Basic Memory entity format.""" # Generate permalink date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d") clean_title = clean_filename(name) - permalink = f"conversations/{date_prefix}-{clean_title}" + permalink = f"{base_path}/{date_prefix}-{clean_title}" # Format content content = format_chat_markdown( @@ -135,6 +135,7 @@ async def process_conversations_json( for chat in conversations: # Convert to entity entity = format_chat_content( + base_path=base_path, name=chat["name"], messages=chat["chat_messages"], created_at=chat["created_at"], @@ -142,7 +143,7 @@ async def process_conversations_json( ) # Write file - file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md" + file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md") await markdown_processor.write_file(file_path, entity) chats_imported += 1 @@ -158,7 +159,7 @@ async def get_markdown_processor() -> MarkdownProcessor: return MarkdownProcessor(entity_parser) -@claude_app.command(name="conversations", help="Import chat conversations from claude.") +@claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.") def import_claude( conversations_json: Annotated[Path, typer.Argument(..., help="Path to conversations.json file")] = "conversations.json", folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", diff --git a/src/basic_memory/cli/commands/import_claude_projects.py b/src/basic_memory/cli/commands/import_claude_projects.py new file mode 100644 index 000000000..3e8896288 --- /dev/null +++ b/src/basic_memory/cli/commands/import_claude_projects.py @@ -0,0 +1,189 @@ +"""Import command for basic-memory CLI to import project data from Claude.ai.""" + +import asyncio +import json +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, List, Annotated, Optional + +import typer +from loguru import logger +from rich.console import Console +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn + +from basic_memory.cli.app import app, import_app, claude_app +from basic_memory.config import config +from basic_memory.markdown import EntityParser, MarkdownProcessor +from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter + +console = Console() + +def clean_filename(text: str) -> str: + """Convert text to safe filename.""" + clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-") + return clean + +def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown: + """Format a project document as a Basic Memory entity.""" + + # Extract timestamps + created_at = doc.get('created_at') or project['created_at'] + modified_at = project['updated_at'] + + # Generate clean names for organization + project_dir = clean_filename(project['name']) + doc_file = clean_filename(doc['filename']) + + # Create entity + entity = EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={ + 'type': 'project_doc', + 'title': doc['filename'], + 'created': created_at, + 'modified': modified_at, + 'permalink': f"{project_dir}/docs/{doc_file}", + 'project_name': project['name'], + 'project_uuid': project['uuid'], + 'doc_uuid': doc['uuid'] + } + ), + content=doc['content'] + ) + + return entity + +def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]: + """Format project prompt template as a Basic Memory entity.""" + + if not project.get('prompt_template'): + return None + + # Extract timestamps + created_at = project['created_at'] + modified_at = project['updated_at'] + + # Generate clean project directory name + project_dir = clean_filename(project['name']) + + # Create entity + entity = EntityMarkdown( + frontmatter=EntityFrontmatter( + metadata={ + 'type': 'prompt_template', + 'title': f"Prompt Template: {project['name']}", + 'created': created_at, + 'modified': modified_at, + 'permalink': f"{project_dir}/prompt-template", + 'project_name': project['name'], + 'project_uuid': project['uuid'] + } + ), + content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}" + ) + + return entity + +async def process_projects_json( + json_path: Path, + base_path: Path, + markdown_processor: MarkdownProcessor +) -> Dict[str, int]: + """Import project data from Claude.ai projects.json format.""" + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + console=console, + ) as progress: + read_task = progress.add_task("Reading project data...", total=None) + + # Read project data + data = json.loads(json_path.read_text()) + progress.update(read_task, total=len(data)) + + # Track import counts + docs_imported = 0 + prompts_imported = 0 + + # Process each project + for project in data: + project_dir = clean_filename(project['name']) + + # Create project directories + docs_dir = base_path / project_dir / "docs" + docs_dir.mkdir(parents=True, exist_ok=True) + + # Import prompt template if it exists + if prompt_entity := format_prompt_markdown(project): + file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md" + await markdown_processor.write_file(file_path, prompt_entity) + prompts_imported += 1 + + # Import project documents + for doc in project.get('docs', []): + entity = format_project_markdown(project, doc) + file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md" + await markdown_processor.write_file(file_path, entity) + docs_imported += 1 + + progress.update(read_task, advance=1) + + return { + "documents": docs_imported, + "prompts": prompts_imported + } + +async def get_markdown_processor() -> MarkdownProcessor: + """Get MarkdownProcessor instance.""" + entity_parser = EntityParser(config.home) + return MarkdownProcessor(entity_parser) + +@claude_app.command(name="projects", help="Import projects from Claude.ai.") +def import_projects( + projects_json: Annotated[Optional[Path], typer.Argument(..., help="Path to projects.json file")] = "projects.json", + base_folder: Annotated[str, typer.Option(help="The base folder to place project files in.")] = "projects", +): + """Import project data from Claude.ai. + + This command will: + 1. Create a directory for each project + 2. Store docs in a docs/ subdirectory + 3. Place prompt template in project root + + After importing, run 'basic-memory sync' to index the new files. + """ + + try: + if projects_json: + if not projects_json.exists(): + typer.echo(f"Error: File not found: {projects_json}", err=True) + raise typer.Exit(1) + + # Get markdown processor + markdown_processor = asyncio.run(get_markdown_processor()) + + # Process the file + base_path = config.home / base_folder if base_folder else config.home + console.print(f"\nImporting projects from {projects_json}...writing to {base_path}") + results = asyncio.run(process_projects_json(projects_json, base_path, markdown_processor)) + + # Show results + console.print( + Panel( + f"[green]Import complete![/green]\n\n" + f"Imported {results['documents']} project documents\n" + f"Imported {results['prompts']} prompt templates", + expand=False, + ) + ) + + console.print("\nRun 'basic-memory sync' to index the new files.") + + except Exception as e: + logger.exception("Import failed") + typer.echo(f"Error during import: {e}", err=True) + raise typer.Exit(1) \ No newline at end of file diff --git a/src/basic_memory/cli/commands/import_memory_json.py b/src/basic_memory/cli/commands/import_memory_json.py index 7f5029d41..ca3f8f633 100644 --- a/src/basic_memory/cli/commands/import_memory_json.py +++ b/src/basic_memory/cli/commands/import_memory_json.py @@ -99,7 +99,7 @@ async def get_markdown_processor() -> MarkdownProcessor: @import_app.command() -def import_json( +def memory_json( json_path: Path = typer.Argument(..., help="Path to memory.json file to import"), ): """Import entities and relations from a memory.json file. diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index 90bc784f9..16cd97a01 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -4,7 +4,15 @@ from basic_memory.utils import setup_logging # pragma: no cover # Register commands -from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp, import_claude_conversations # noqa: F401 # pragma: no cover +from basic_memory.cli.commands import ( + status, + sync, + db, + import_memory_json, + mcp, + import_claude_conversations, + import_claude_projects, +) # noqa: F401 # pragma: no cover # Set up logging when module is imported diff --git a/tests/cli/test_import_memory_json.py b/tests/cli/test_import_memory_json.py index 4f146dfb2..76491385b 100644 --- a/tests/cli/test_import_memory_json.py +++ b/tests/cli/test_import_memory_json.py @@ -4,7 +4,7 @@ import pytest from typer.testing import CliRunner -from basic_memory.cli.app import app +from basic_memory.cli.app import import_app from basic_memory.cli.commands import import_memory_json from basic_memory.markdown import EntityParser, MarkdownProcessor @@ -72,7 +72,7 @@ async def test_get_markdown_processor(tmp_path, monkeypatch): def test_import_json_command_file_not_found(tmp_path): """Test error handling for nonexistent file.""" nonexistent = tmp_path / "nonexistent.json" - result = runner.invoke(app, ["import-json", str(nonexistent)]) + result = runner.invoke(import_app, ["memory-json", str(nonexistent)]) assert result.exit_code == 1 assert "File not found" in result.output @@ -83,7 +83,7 @@ def test_import_json_command_success(tmp_path, sample_json_file, monkeypatch): monkeypatch.setenv("HOME", str(tmp_path)) # Run import - result = runner.invoke(app, ["import-json", str(sample_json_file)]) + result = runner.invoke(import_app, ["memory-json", str(sample_json_file)]) assert result.exit_code == 0 assert "Import complete" in result.output assert "Created 1 entities" in result.output @@ -96,7 +96,7 @@ def test_import_json_command_invalid_json(tmp_path): invalid_file = tmp_path / "invalid.json" invalid_file.write_text("not json") - result = runner.invoke(app, ["import-json", str(invalid_file)]) + result = runner.invoke(import_app, ["memory-json", str(invalid_file)]) assert result.exit_code == 1 assert "Error during import" in result.output @@ -129,6 +129,6 @@ def test_import_json_command_handle_old_format(tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) # Run import - result = runner.invoke(app, ["import-json", str(json_file)]) + result = runner.invoke(import_app, ["memory-json", str(json_file)]) assert result.exit_code == 0 assert "Import complete" in result.output From 90072166f09a8b0e8ed12326dd4bf96fd8f1024c Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 12:21:23 -0600 Subject: [PATCH 7/8] add tests for importing claude conversations --- tests/cli/test_import_claude_conversations.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/cli/test_import_claude_conversations.py diff --git a/tests/cli/test_import_claude_conversations.py b/tests/cli/test_import_claude_conversations.py new file mode 100644 index 000000000..dc3587f5e --- /dev/null +++ b/tests/cli/test_import_claude_conversations.py @@ -0,0 +1,199 @@ +"""Tests for import_claude command (chat conversations).""" + +import json +import pytest +from typer.testing import CliRunner +from pathlib import Path + +from basic_memory.cli.app import import_app, claude_app, app +from basic_memory.cli.commands import import_claude_conversations as import_claude +from basic_memory.config import config +from basic_memory.markdown import EntityParser, MarkdownProcessor + +# Set up CLI runner +runner = CliRunner() + + +@pytest.fixture +def sample_conversation(): + """Sample conversation data for testing.""" + return { + "uuid": "test-uuid", + "name": "Test Conversation", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "updated_at": "2025-01-05T20:56:39.477600+00:00", + "chat_messages": [ + { + "uuid": "msg-1", + "text": "Hello, this is a test", + "sender": "human", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "content": [ + { + "type": "text", + "text": "Hello, this is a test" + } + ] + }, + { + "uuid": "msg-2", + "text": "Response to test", + "sender": "assistant", + "created_at": "2025-01-05T20:55:40.123456+00:00", + "content": [ + { + "type": "text", + "text": "Response to test" + } + ] + } + ] + } + + +@pytest.fixture +def sample_conversations_json(tmp_path, sample_conversation): + """Create a sample conversations.json file.""" + json_file = tmp_path / "conversations.json" + with open(json_file, "w") as f: + json.dump([sample_conversation], f) + return json_file + + +@pytest.mark.asyncio +async def test_process_chat_json(tmp_path, sample_conversations_json): + """Test importing conversations from JSON.""" + entity_parser = EntityParser(tmp_path) + processor = MarkdownProcessor(entity_parser) + + results = await import_claude.process_conversations_json( + sample_conversations_json, tmp_path, processor + ) + + assert results["conversations"] == 1 + assert results["messages"] == 2 + + # Check conversation file + conv_path = tmp_path / "20250105-test-conversation.md" + assert conv_path.exists() + content = conv_path.read_text() + + # Check content formatting + assert "### Human" in content + assert "Hello, this is a test" in content + assert "### Assistant" in content + assert "Response to test" in content + + +def test_import_conversations_command_file_not_found(tmp_path): + """Test error handling for nonexistent file.""" + nonexistent = tmp_path / "nonexistent.json" + result = runner.invoke( + app, + ["import", "claude", "conversations", str(nonexistent)] + ) + assert result.exit_code == 1 + assert "File not found" in result.output + + +def test_import_conversations_command_success(tmp_path, sample_conversations_json, monkeypatch): + """Test successful conversation import via command.""" + # Set up test environment + monkeypatch.setenv("HOME", str(tmp_path)) + + # Run import + result = runner.invoke( + app, + ["import", "claude", "conversations", str(sample_conversations_json)] + ) + assert result.exit_code == 0 + assert "Import complete" in result.output + assert "Imported 1 conversations" in result.output + assert "Containing 2 messages" in result.output + + +def test_import_conversations_command_invalid_json(tmp_path): + """Test error handling for invalid JSON.""" + # Create invalid JSON file + invalid_file = tmp_path / "invalid.json" + invalid_file.write_text("not json") + + result = runner.invoke( + app, + ["import", "claude", "conversations", str(invalid_file)] + ) + assert result.exit_code == 1 + assert "Error during import" in result.output + + +def test_import_conversations_with_custom_folder(tmp_path, sample_conversations_json, monkeypatch): + """Test import with custom conversations folder.""" + # Set up test environment + config.home = tmp_path + conversations_folder = "chats" + + # Run import + result = runner.invoke( + app, + ["import", "claude", "conversations", + str(sample_conversations_json), + "--folder", conversations_folder + ] + ) + assert result.exit_code == 0 + + # Check files in custom folder + conv_path = tmp_path / conversations_folder / "20250105-test-conversation.md" + assert conv_path.exists() + + +def test_import_conversation_with_attachments(tmp_path): + """Test importing conversation with attachments.""" + # Create conversation with attachments + conversation = { + "uuid": "test-uuid", + "name": "Test With Attachments", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "updated_at": "2025-01-05T20:56:39.477600+00:00", + "chat_messages": [ + { + "uuid": "msg-1", + "text": "Here's a file", + "sender": "human", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "content": [ + { + "type": "text", + "text": "Here's a file" + } + ], + "attachments": [ + { + "file_name": "test.txt", + "extracted_content": "Test file content" + } + ] + } + ] + } + + json_file = tmp_path / "with_attachments.json" + with open(json_file, "w") as f: + json.dump([conversation], f) + + # Set up environment + config.home = tmp_path + + # Run import + result = runner.invoke( + app, + ["import", "claude", "conversations", str(json_file)] + ) + assert result.exit_code == 0 + + # Check attachment formatting + conv_path = tmp_path / "conversations/20250105-test-with-attachments.md" + content = conv_path.read_text() + assert "**Attachment: test.txt**" in content + assert "```" in content + assert "Test file content" in content \ No newline at end of file From e5d266e382e4674a65ee1efafef7397af8da98a9 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sun, 16 Feb 2025 12:41:29 -0600 Subject: [PATCH 8/8] feat: claude.ai data import --- README.md | 73 +++++++- src/basic_memory/cli/app.py | 4 +- .../commands/import_claude_conversations.py | 20 +- .../cli/commands/import_claude_projects.py | 140 +++++++------- .../cli/commands/import_memory_json.py | 8 +- src/basic_memory/cli/main.py | 4 +- tests/cli/test_import_claude_conversations.py | 77 +++----- tests/cli/test_import_claude_projects.py | 173 ++++++++++++++++++ uv.lock | 2 +- 9 files changed, 367 insertions(+), 134 deletions(-) create mode 100644 tests/cli/test_import_claude_projects.py diff --git a/README.md b/README.md index cbd9d7451..8fcdf7edc 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,77 @@ Basic Memory is built on some key ideas: - Simple text patterns can capture rich meaning - Local-first doesn't mean feature-poor +## Importing data + +Basic memory has cli commands to import data from several formats into Markdown files + +### Claude.ai + +First, request an export of your data from your Claude account. The data will be emailed to you in several files, +including +`conversations.json` and `projects.json`. + +Import Claude.ai conversation data + +```bash + basic-memory import claude conversations conversations.json +``` + +The conversations will be turned into Markdown files and placed in the "conversations" folder by default (this can be +changed with the --folder arg). + +Example: + +```bash +Importing chats from conversations.json...writing to .../basic-memory + Reading chat data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% +╭────────────────────────────╮ +│ Import complete! │ +│ │ +│ Imported 307 conversations │ +│ Containing 7769 messages │ +╰────────────────────────────╯ +``` + +Next, you can run the `sync` command to import the data into basic-memory + +```bash +basic-memory sync +``` + +You can also import project data from Claude.ai + +```bash +➜ basic-memory import claude projects +Importing projects from projects.json...writing to .../basic-memory/projects + Reading project data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% +╭────────────────────────────────╮ +│ Import complete! │ +│ │ +│ Imported 101 project documents │ +│ Imported 32 prompt templates │ +╰────────────────────────────────╯ + +Run 'basic-memory sync' to index the new files. +``` + +### Chat Gpt + +### Memory json + +```bash +➜ basic-memory import memory-json +Importing from memory.json...writing to .../basic-memory + Reading memory.json... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% + Creating entities... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% +╭──────────────────────╮ +│ Import complete! │ +│ │ +│ Created 126 entities │ +│ Added 252 relations │ +╰──────────────────────╯ +``` + ## License -AGPL-3.0 \ No newline at end of file +AGPL-3.0 diff --git a/src/basic_memory/cli/app.py b/src/basic_memory/cli/app.py index 2fb907909..dbd806862 100644 --- a/src/basic_memory/cli/app.py +++ b/src/basic_memory/cli/app.py @@ -10,11 +10,11 @@ asyncio.run(db.run_migrations(config)) -app = typer.Typer() +app = typer.Typer(name="basic-memory") import_app = typer.Typer() app.add_typer(import_app, name="import") claude_app = typer.Typer() -import_app.add_typer(claude_app, name="claude") \ No newline at end of file +import_app.add_typer(claude_app, name="claude") diff --git a/src/basic_memory/cli/commands/import_claude_conversations.py b/src/basic_memory/cli/commands/import_claude_conversations.py index 40a7720e2..3c6ea814a 100644 --- a/src/basic_memory/cli/commands/import_claude_conversations.py +++ b/src/basic_memory/cli/commands/import_claude_conversations.py @@ -4,7 +4,7 @@ import json from datetime import datetime from pathlib import Path -from typing import Dict, Any, List, Annotated, Optional +from typing import Dict, Any, List, Annotated import typer from loguru import logger @@ -12,7 +12,7 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn -from basic_memory.cli.app import app, import_app, claude_app +from basic_memory.cli.app import claude_app from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter @@ -74,7 +74,7 @@ def format_chat_markdown( def format_chat_content( - base_path: str, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str + base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str ) -> EntityMarkdown: """Convert chat messages to Basic Memory entity format.""" @@ -161,8 +161,12 @@ async def get_markdown_processor() -> MarkdownProcessor: @claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.") def import_claude( - conversations_json: Annotated[Path, typer.Argument(..., help="Path to conversations.json file")] = "conversations.json", - folder: Annotated[str, typer.Option(help="The folder to place the files in.")] = "conversations", + conversations_json: Annotated[ + Path, typer.Argument(..., help="Path to conversations.json file") + ] = Path("conversations.json"), + folder: Annotated[ + str, typer.Option(help="The folder to place the files in.") + ] = "conversations", ): """Import chat conversations from conversations2.json format. @@ -178,14 +182,16 @@ def import_claude( if not conversations_json.exists(): typer.echo(f"Error: File not found: {conversations_json}", err=True) raise typer.Exit(1) - + # Get markdown processor markdown_processor = asyncio.run(get_markdown_processor()) # Process the file base_path = config.home / folder console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}") - results = asyncio.run(process_conversations_json(conversations_json, base_path, markdown_processor)) + results = asyncio.run( + process_conversations_json(conversations_json, base_path, markdown_processor) + ) # Show results console.print( diff --git a/src/basic_memory/cli/commands/import_claude_projects.py b/src/basic_memory/cli/commands/import_claude_projects.py index 3e8896288..5e2df7e20 100644 --- a/src/basic_memory/cli/commands/import_claude_projects.py +++ b/src/basic_memory/cli/commands/import_claude_projects.py @@ -2,9 +2,8 @@ import asyncio import json -from datetime import datetime from pathlib import Path -from typing import Dict, Any, List, Annotated, Optional +from typing import Dict, Any, Annotated, Optional import typer from loguru import logger @@ -12,86 +11,88 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn -from basic_memory.cli.app import app, import_app, claude_app +from basic_memory.cli.app import claude_app from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter console = Console() + def clean_filename(text: str) -> str: """Convert text to safe filename.""" clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-") return clean + def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown: """Format a project document as a Basic Memory entity.""" - + # Extract timestamps - created_at = doc.get('created_at') or project['created_at'] - modified_at = project['updated_at'] - + created_at = doc.get("created_at") or project["created_at"] + modified_at = project["updated_at"] + # Generate clean names for organization - project_dir = clean_filename(project['name']) - doc_file = clean_filename(doc['filename']) - + project_dir = clean_filename(project["name"]) + doc_file = clean_filename(doc["filename"]) + # Create entity entity = EntityMarkdown( frontmatter=EntityFrontmatter( metadata={ - 'type': 'project_doc', - 'title': doc['filename'], - 'created': created_at, - 'modified': modified_at, - 'permalink': f"{project_dir}/docs/{doc_file}", - 'project_name': project['name'], - 'project_uuid': project['uuid'], - 'doc_uuid': doc['uuid'] + "type": "project_doc", + "title": doc["filename"], + "created": created_at, + "modified": modified_at, + "permalink": f"{project_dir}/docs/{doc_file}", + "project_name": project["name"], + "project_uuid": project["uuid"], + "doc_uuid": doc["uuid"], } ), - content=doc['content'] + content=doc["content"], ) - + return entity + def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]: """Format project prompt template as a Basic Memory entity.""" - - if not project.get('prompt_template'): + + if not project.get("prompt_template"): return None - + # Extract timestamps - created_at = project['created_at'] - modified_at = project['updated_at'] - + created_at = project["created_at"] + modified_at = project["updated_at"] + # Generate clean project directory name - project_dir = clean_filename(project['name']) - + project_dir = clean_filename(project["name"]) + # Create entity entity = EntityMarkdown( frontmatter=EntityFrontmatter( metadata={ - 'type': 'prompt_template', - 'title': f"Prompt Template: {project['name']}", - 'created': created_at, - 'modified': modified_at, - 'permalink': f"{project_dir}/prompt-template", - 'project_name': project['name'], - 'project_uuid': project['uuid'] + "type": "prompt_template", + "title": f"Prompt Template: {project['name']}", + "created": created_at, + "modified": modified_at, + "permalink": f"{project_dir}/prompt-template", + "project_name": project["name"], + "project_uuid": project["uuid"], } ), - content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}" + content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}", ) - + return entity + async def process_projects_json( - json_path: Path, - base_path: Path, - markdown_processor: MarkdownProcessor + json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor ) -> Dict[str, int]: """Import project data from Claude.ai projects.json format.""" - + with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -100,77 +101,82 @@ async def process_projects_json( console=console, ) as progress: read_task = progress.add_task("Reading project data...", total=None) - + # Read project data data = json.loads(json_path.read_text()) progress.update(read_task, total=len(data)) - + # Track import counts docs_imported = 0 prompts_imported = 0 - + # Process each project for project in data: - project_dir = clean_filename(project['name']) - + project_dir = clean_filename(project["name"]) + # Create project directories docs_dir = base_path / project_dir / "docs" docs_dir.mkdir(parents=True, exist_ok=True) - + # Import prompt template if it exists if prompt_entity := format_prompt_markdown(project): file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md" await markdown_processor.write_file(file_path, prompt_entity) prompts_imported += 1 - + # Import project documents - for doc in project.get('docs', []): + for doc in project.get("docs", []): entity = format_project_markdown(project, doc) file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md" await markdown_processor.write_file(file_path, entity) docs_imported += 1 - + progress.update(read_task, advance=1) - - return { - "documents": docs_imported, - "prompts": prompts_imported - } + + return {"documents": docs_imported, "prompts": prompts_imported} + async def get_markdown_processor() -> MarkdownProcessor: """Get MarkdownProcessor instance.""" entity_parser = EntityParser(config.home) return MarkdownProcessor(entity_parser) + @claude_app.command(name="projects", help="Import projects from Claude.ai.") def import_projects( - projects_json: Annotated[Optional[Path], typer.Argument(..., help="Path to projects.json file")] = "projects.json", - base_folder: Annotated[str, typer.Option(help="The base folder to place project files in.")] = "projects", + projects_json: Annotated[Path, typer.Argument(..., help="Path to projects.json file")] = Path( + "projects.json" + ), + base_folder: Annotated[ + str, typer.Option(help="The base folder to place project files in.") + ] = "projects", ): """Import project data from Claude.ai. - + This command will: 1. Create a directory for each project 2. Store docs in a docs/ subdirectory 3. Place prompt template in project root - + After importing, run 'basic-memory sync' to index the new files. """ - + try: if projects_json: if not projects_json.exists(): typer.echo(f"Error: File not found: {projects_json}", err=True) raise typer.Exit(1) - + # Get markdown processor markdown_processor = asyncio.run(get_markdown_processor()) - + # Process the file base_path = config.home / base_folder if base_folder else config.home console.print(f"\nImporting projects from {projects_json}...writing to {base_path}") - results = asyncio.run(process_projects_json(projects_json, base_path, markdown_processor)) - + results = asyncio.run( + process_projects_json(projects_json, base_path, markdown_processor) + ) + # Show results console.print( Panel( @@ -180,10 +186,10 @@ def import_projects( expand=False, ) ) - + console.print("\nRun 'basic-memory sync' to index the new files.") - + except Exception as e: logger.exception("Import failed") typer.echo(f"Error during import: {e}", err=True) - raise typer.Exit(1) \ No newline at end of file + raise typer.Exit(1) diff --git a/src/basic_memory/cli/commands/import_memory_json.py b/src/basic_memory/cli/commands/import_memory_json.py index ca3f8f633..826fcf738 100644 --- a/src/basic_memory/cli/commands/import_memory_json.py +++ b/src/basic_memory/cli/commands/import_memory_json.py @@ -3,7 +3,7 @@ import asyncio import json from pathlib import Path -from typing import Dict, Any, List +from typing import Dict, Any, List, Annotated import typer from loguru import logger @@ -11,7 +11,7 @@ from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn -from basic_memory.cli.app import app, import_app +from basic_memory.cli.app import import_app from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation @@ -100,7 +100,9 @@ async def get_markdown_processor() -> MarkdownProcessor: @import_app.command() def memory_json( - json_path: Path = typer.Argument(..., help="Path to memory.json file to import"), + json_path: Annotated[Path, typer.Argument(..., help="Path to memory.json file")] = Path( + "memory.json" + ), ): """Import entities and relations from a memory.json file. diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index 16cd97a01..678b38d02 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -4,7 +4,7 @@ from basic_memory.utils import setup_logging # pragma: no cover # Register commands -from basic_memory.cli.commands import ( +from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover status, sync, db, @@ -12,7 +12,7 @@ mcp, import_claude_conversations, import_claude_projects, -) # noqa: F401 # pragma: no cover +) # Set up logging when module is imported diff --git a/tests/cli/test_import_claude_conversations.py b/tests/cli/test_import_claude_conversations.py index dc3587f5e..1479c7413 100644 --- a/tests/cli/test_import_claude_conversations.py +++ b/tests/cli/test_import_claude_conversations.py @@ -3,9 +3,8 @@ import json import pytest from typer.testing import CliRunner -from pathlib import Path -from basic_memory.cli.app import import_app, claude_app, app +from basic_memory.cli.app import app from basic_memory.cli.commands import import_claude_conversations as import_claude from basic_memory.config import config from basic_memory.markdown import EntityParser, MarkdownProcessor @@ -28,26 +27,16 @@ def sample_conversation(): "text": "Hello, this is a test", "sender": "human", "created_at": "2025-01-05T20:55:32.499880+00:00", - "content": [ - { - "type": "text", - "text": "Hello, this is a test" - } - ] + "content": [{"type": "text", "text": "Hello, this is a test"}], }, { "uuid": "msg-2", "text": "Response to test", "sender": "assistant", "created_at": "2025-01-05T20:55:40.123456+00:00", - "content": [ - { - "type": "text", - "text": "Response to test" - } - ] - } - ] + "content": [{"type": "text", "text": "Response to test"}], + }, + ], } @@ -56,7 +45,7 @@ def sample_conversations_json(tmp_path, sample_conversation): """Create a sample conversations.json file.""" json_file = tmp_path / "conversations.json" with open(json_file, "w") as f: - json.dump([sample_conversation], f) + json.dump([sample_conversation], f) return json_file @@ -77,7 +66,7 @@ async def test_process_chat_json(tmp_path, sample_conversations_json): conv_path = tmp_path / "20250105-test-conversation.md" assert conv_path.exists() content = conv_path.read_text() - + # Check content formatting assert "### Human" in content assert "Hello, this is a test" in content @@ -88,10 +77,7 @@ async def test_process_chat_json(tmp_path, sample_conversations_json): def test_import_conversations_command_file_not_found(tmp_path): """Test error handling for nonexistent file.""" nonexistent = tmp_path / "nonexistent.json" - result = runner.invoke( - app, - ["import", "claude", "conversations", str(nonexistent)] - ) + result = runner.invoke(app, ["import", "claude", "conversations", str(nonexistent)]) assert result.exit_code == 1 assert "File not found" in result.output @@ -103,8 +89,7 @@ def test_import_conversations_command_success(tmp_path, sample_conversations_jso # Run import result = runner.invoke( - app, - ["import", "claude", "conversations", str(sample_conversations_json)] + app, ["import", "claude", "conversations", str(sample_conversations_json)] ) assert result.exit_code == 0 assert "Import complete" in result.output @@ -118,10 +103,7 @@ def test_import_conversations_command_invalid_json(tmp_path): invalid_file = tmp_path / "invalid.json" invalid_file.write_text("not json") - result = runner.invoke( - app, - ["import", "claude", "conversations", str(invalid_file)] - ) + result = runner.invoke(app, ["import", "claude", "conversations", str(invalid_file)]) assert result.exit_code == 1 assert "Error during import" in result.output @@ -135,13 +117,17 @@ def test_import_conversations_with_custom_folder(tmp_path, sample_conversations_ # Run import result = runner.invoke( app, - ["import", "claude", "conversations", + [ + "import", + "claude", + "conversations", str(sample_conversations_json), - "--folder", conversations_folder - ] + "--folder", + conversations_folder, + ], ) assert result.exit_code == 0 - + # Check files in custom folder conv_path = tmp_path / conversations_folder / "20250105-test-conversation.md" assert conv_path.exists() @@ -161,22 +147,14 @@ def test_import_conversation_with_attachments(tmp_path): "text": "Here's a file", "sender": "human", "created_at": "2025-01-05T20:55:32.499880+00:00", - "content": [ - { - "type": "text", - "text": "Here's a file" - } - ], + "content": [{"type": "text", "text": "Here's a file"}], "attachments": [ - { - "file_name": "test.txt", - "extracted_content": "Test file content" - } - ] + {"file_name": "test.txt", "extracted_content": "Test file content"} + ], } - ] + ], } - + json_file = tmp_path / "with_attachments.json" with open(json_file, "w") as f: json.dump([conversation], f) @@ -185,15 +163,12 @@ def test_import_conversation_with_attachments(tmp_path): config.home = tmp_path # Run import - result = runner.invoke( - app, - ["import", "claude", "conversations", str(json_file)] - ) + result = runner.invoke(app, ["import", "claude", "conversations", str(json_file)]) assert result.exit_code == 0 - + # Check attachment formatting conv_path = tmp_path / "conversations/20250105-test-with-attachments.md" content = conv_path.read_text() assert "**Attachment: test.txt**" in content assert "```" in content - assert "Test file content" in content \ No newline at end of file + assert "Test file content" in content diff --git a/tests/cli/test_import_claude_projects.py b/tests/cli/test_import_claude_projects.py new file mode 100644 index 000000000..3c5b29ba6 --- /dev/null +++ b/tests/cli/test_import_claude_projects.py @@ -0,0 +1,173 @@ +"""Tests for import_claude_projects command.""" + +import json +import pytest +from typer.testing import CliRunner + +from basic_memory.cli.app import app +from basic_memory.cli.commands import import_claude_projects +from basic_memory.config import config +from basic_memory.markdown import EntityParser, MarkdownProcessor + +# Set up CLI runner +runner = CliRunner() + + +@pytest.fixture +def sample_project(): + """Sample project data for testing.""" + return { + "uuid": "test-uuid", + "name": "Test Project", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "updated_at": "2025-01-05T20:56:39.477600+00:00", + "prompt_template": "# Test Prompt\n\nThis is a test prompt.", + "docs": [ + { + "uuid": "doc-uuid-1", + "filename": "Test Document", + "content": "# Test Document\n\nThis is test content.", + "created_at": "2025-01-05T20:56:39.477600+00:00", + }, + { + "uuid": "doc-uuid-2", + "filename": "Another Document", + "content": "# Another Document\n\nMore test content.", + "created_at": "2025-01-05T20:56:39.477600+00:00", + }, + ], + } + + +@pytest.fixture +def sample_projects_json(tmp_path, sample_project): + """Create a sample projects.json file.""" + json_file = tmp_path / "projects.json" + with open(json_file, "w") as f: + json.dump([sample_project], f) + return json_file + + +@pytest.mark.asyncio +async def test_process_projects_json(tmp_path, sample_projects_json): + """Test importing projects from JSON.""" + entity_parser = EntityParser(tmp_path) + processor = MarkdownProcessor(entity_parser) + + results = await import_claude_projects.process_projects_json( + sample_projects_json, tmp_path, processor + ) + + assert results["documents"] == 2 + assert results["prompts"] == 1 + + # Check project directory structure + project_dir = tmp_path / "test-project" + assert project_dir.exists() + assert (project_dir / "docs").exists() + assert (project_dir / "prompt-template.md").exists() + + # Check document files + doc1 = project_dir / "docs/test-document.md" + assert doc1.exists() + content1 = doc1.read_text() + assert "# Test Document" in content1 + assert "This is test content" in content1 + + # Check prompt template + prompt = project_dir / "prompt-template.md" + assert prompt.exists() + prompt_content = prompt.read_text() + assert "# Test Prompt" in prompt_content + assert "This is a test prompt" in prompt_content + + +def test_import_projects_command_file_not_found(tmp_path): + """Test error handling for nonexistent file.""" + nonexistent = tmp_path / "nonexistent.json" + result = runner.invoke(app, ["import", "claude", "projects", str(nonexistent)]) + assert result.exit_code == 1 + assert "File not found" in result.output + + +def test_import_projects_command_success(tmp_path, sample_projects_json, monkeypatch): + """Test successful project import via command.""" + # Set up test environment + config.home = tmp_path + + # Run import + result = runner.invoke(app, ["import", "claude", "projects", str(sample_projects_json)]) + assert result.exit_code == 0 + assert "Import complete" in result.output + assert "Imported 2 project documents" in result.output + assert "Imported 1 prompt templates" in result.output + + +def test_import_projects_command_invalid_json(tmp_path): + """Test error handling for invalid JSON.""" + # Create invalid JSON file + invalid_file = tmp_path / "invalid.json" + invalid_file.write_text("not json") + + result = runner.invoke(app, ["import", "claude", "projects", str(invalid_file)]) + assert result.exit_code == 1 + assert "Error during import" in result.output + + +def test_import_projects_with_base_folder(tmp_path, sample_projects_json, monkeypatch): + """Test import with custom base folder.""" + # Set up test environment + config.home = tmp_path + base_folder = "claude-exports" + + # Run import + result = runner.invoke( + app, + [ + "import", + "claude", + "projects", + str(sample_projects_json), + "--base-folder", + base_folder, + ], + ) + assert result.exit_code == 0 + + # Check files in base folder + project_dir = tmp_path / base_folder / "test-project" + assert project_dir.exists() + assert (project_dir / "docs").exists() + assert (project_dir / "prompt-template.md").exists() + + +def test_import_project_without_prompt(tmp_path): + """Test importing project without prompt template.""" + # Create project without prompt + project = { + "uuid": "test-uuid", + "name": "No Prompt Project", + "created_at": "2025-01-05T20:55:32.499880+00:00", + "updated_at": "2025-01-05T20:56:39.477600+00:00", + "docs": [ + { + "uuid": "doc-uuid-1", + "filename": "Test Document", + "content": "# Test Document\n\nContent.", + "created_at": "2025-01-05T20:56:39.477600+00:00", + } + ], + } + + json_file = tmp_path / "no_prompt.json" + with open(json_file, "w") as f: + json.dump([project], f) + + # Set up environment + config.home = tmp_path + + # Run import + result = runner.invoke(app, ["import", "claude", "projects", str(json_file)]) + assert result.exit_code == 0 + assert "Imported 1 project documents" in result.output + assert "Imported 0 prompt templates" in result.output diff --git a/uv.lock b/uv.lock index 578e9be60..4932a03a1 100644 --- a/uv.lock +++ b/uv.lock @@ -70,7 +70,7 @@ wheels = [ [[package]] name = "basic-memory" -version = "0.2.21" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "aiosqlite" },