From 0223d8a244a02a9acac0263adaa356b2b155e3d1 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 21:53:31 +0700 Subject: [PATCH 01/17] add cli usage --- .gitignore | 61 ++- codewiki/README.md | 138 +++++++ codewiki/__init__.py | 14 + codewiki/__main__.py | 8 + codewiki/cli/__init__.py | 4 + codewiki/cli/adapters/__init__.py | 4 + codewiki/cli/adapters/doc_generator.py | 245 +++++++++++ codewiki/cli/commands/__init__.py | 4 + codewiki/cli/commands/config.py | 381 ++++++++++++++++++ codewiki/cli/commands/generate.py | 256 ++++++++++++ codewiki/cli/config_manager.py | 226 +++++++++++ codewiki/cli/git_manager.py | 227 +++++++++++ codewiki/cli/html_generator.py | 177 ++++++++ codewiki/cli/main.py | 56 +++ codewiki/cli/models/__init__.py | 4 + codewiki/cli/models/config.py | 102 +++++ codewiki/cli/models/job.py | 156 +++++++ codewiki/cli/utils/__init__.py | 4 + codewiki/cli/utils/api_errors.py | 140 +++++++ codewiki/cli/utils/errors.py | 113 ++++++ codewiki/cli/utils/fs.py | 190 +++++++++ codewiki/cli/utils/instructions.py | 179 ++++++++ codewiki/cli/utils/logging.py | 85 ++++ codewiki/cli/utils/progress.py | 222 ++++++++++ codewiki/cli/utils/repo_validator.py | 184 +++++++++ codewiki/cli/utils/validation.py | 231 +++++++++++ codewiki/py.typed | 2 + run_web_app.py => codewiki/run_web_app.py | 0 codewiki/src/__init__.py | 2 + codewiki/src/be/__init__.py | 2 + .../src}/be/agent_orchestrator.py | 72 ++-- codewiki/src/be/agent_tools/__init__.py | 2 + {src => codewiki/src}/be/agent_tools/deps.py | 6 +- .../generate_sub_module_documentations.py | 19 +- .../be/agent_tools/read_code_components.py | 2 +- .../src}/be/agent_tools/str_replace_editor.py | 0 {src => codewiki/src}/be/cluster_modules.py | 15 +- .../src/be/dependency_analyzer/__init__.py | 21 + .../dependency_analyzer/analysis/__init__.py | 0 .../analysis/analysis_service.py | 12 +- .../analysis/call_graph_analyzer.py | 20 +- .../dependency_analyzer/analysis/cloning.py | 0 .../analysis/repo_analyzer.py | 2 +- .../dependency_analyzer/analyzers/__init__.py | 0 .../be/dependency_analyzer/analyzers/c.py | 2 +- .../be/dependency_analyzer/analyzers/cpp.py | 2 +- .../dependency_analyzer/analyzers/csharp.py | 2 +- .../be/dependency_analyzer/analyzers/java.py | 2 +- .../analyzers/javascript.py | 2 +- .../dependency_analyzer/analyzers/python.py | 2 +- .../analyzers/typescript.py | 2 +- .../src}/be/dependency_analyzer/ast_parser.py | 68 +--- .../dependency_graphs_builder.py | 8 +- .../be/dependency_analyzer/models/__init__.py | 0 .../be/dependency_analyzer/models/analysis.py | 2 +- .../be/dependency_analyzer/models/core.py | 0 .../src}/be/dependency_analyzer/topo_sort.py | 2 +- .../be/dependency_analyzer/utils/__init__.py | 0 .../utils/logging_config.py | 0 .../be/dependency_analyzer/utils/patterns.py | 0 .../be/dependency_analyzer/utils/security.py | 0 .../src}/be/documentation_generator.py | 23 +- codewiki/src/be/llm_services.py | 86 ++++ {src => codewiki/src}/be/main.py | 4 +- {src => codewiki/src}/be/prompt_template.py | 2 +- {src => codewiki/src}/be/utils.py | 0 codewiki/src/config.py | 114 ++++++ {src => codewiki/src}/fe/__init__.py | 0 {src => codewiki/src}/fe/background_worker.py | 20 +- {src => codewiki/src}/fe/cache_manager.py | 2 +- {src => codewiki/src}/fe/config.py | 0 {src => codewiki/src}/fe/github_processor.py | 0 {src => codewiki/src}/fe/models.py | 0 {src => codewiki/src}/fe/routes.py | 2 +- {src => codewiki/src}/fe/template_utils.py | 0 {src => codewiki/src}/fe/templates.py | 0 {src => codewiki/src}/fe/visualise_docs.py | 2 +- {src => codewiki/src}/fe/web_app.py | 0 {src => codewiki/src}/utils.py | 0 codewiki/templates/github_pages/.gitkeep | 3 + codewiki/templates/github_pages/app.js | 179 ++++++++ codewiki/templates/github_pages/index.html | 60 +++ codewiki/templates/github_pages/styles.css | 244 +++++++++++ pyproject.toml | 123 ++++++ src/be/dependency_analyzer/__init__.py | 21 - src/be/llm_services.py | 50 --- src/config.py | 46 --- 87 files changed, 4367 insertions(+), 296 deletions(-) create mode 100644 codewiki/README.md create mode 100644 codewiki/__init__.py create mode 100644 codewiki/__main__.py create mode 100644 codewiki/cli/__init__.py create mode 100644 codewiki/cli/adapters/__init__.py create mode 100644 codewiki/cli/adapters/doc_generator.py create mode 100644 codewiki/cli/commands/__init__.py create mode 100644 codewiki/cli/commands/config.py create mode 100644 codewiki/cli/commands/generate.py create mode 100644 codewiki/cli/config_manager.py create mode 100644 codewiki/cli/git_manager.py create mode 100644 codewiki/cli/html_generator.py create mode 100644 codewiki/cli/main.py create mode 100644 codewiki/cli/models/__init__.py create mode 100644 codewiki/cli/models/config.py create mode 100644 codewiki/cli/models/job.py create mode 100644 codewiki/cli/utils/__init__.py create mode 100644 codewiki/cli/utils/api_errors.py create mode 100644 codewiki/cli/utils/errors.py create mode 100644 codewiki/cli/utils/fs.py create mode 100644 codewiki/cli/utils/instructions.py create mode 100644 codewiki/cli/utils/logging.py create mode 100644 codewiki/cli/utils/progress.py create mode 100644 codewiki/cli/utils/repo_validator.py create mode 100644 codewiki/cli/utils/validation.py create mode 100644 codewiki/py.typed rename run_web_app.py => codewiki/run_web_app.py (100%) create mode 100644 codewiki/src/__init__.py create mode 100644 codewiki/src/be/__init__.py rename {src => codewiki/src}/be/agent_orchestrator.py (67%) create mode 100644 codewiki/src/be/agent_tools/__init__.py rename {src => codewiki/src}/be/agent_tools/deps.py (63%) rename {src => codewiki/src}/be/agent_tools/generate_sub_module_documentations.py (82%) rename {src => codewiki/src}/be/agent_tools/read_code_components.py (94%) rename {src => codewiki/src}/be/agent_tools/str_replace_editor.py (100%) rename {src => codewiki/src}/be/cluster_modules.py (90%) create mode 100644 codewiki/src/be/dependency_analyzer/__init__.py rename {src => codewiki/src}/be/dependency_analyzer/analysis/__init__.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/analysis/analysis_service.py (96%) rename {src => codewiki/src}/be/dependency_analyzer/analysis/call_graph_analyzer.py (95%) rename {src => codewiki/src}/be/dependency_analyzer/analysis/cloning.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/analysis/repo_analyzer.py (97%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/__init__.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/c.py (98%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/cpp.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/csharp.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/java.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/javascript.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/python.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/analyzers/typescript.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/ast_parser.py (70%) rename {src => codewiki/src}/be/dependency_analyzer/dependency_graphs_builder.py (92%) rename {src => codewiki/src}/be/dependency_analyzer/models/__init__.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/models/analysis.py (85%) rename {src => codewiki/src}/be/dependency_analyzer/models/core.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/topo_sort.py (99%) rename {src => codewiki/src}/be/dependency_analyzer/utils/__init__.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/utils/logging_config.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/utils/patterns.py (100%) rename {src => codewiki/src}/be/dependency_analyzer/utils/security.py (100%) rename {src => codewiki/src}/be/documentation_generator.py (95%) create mode 100644 codewiki/src/be/llm_services.py rename {src => codewiki/src}/be/main.py (93%) rename {src => codewiki/src}/be/prompt_template.py (99%) rename {src => codewiki/src}/be/utils.py (100%) create mode 100644 codewiki/src/config.py rename {src => codewiki/src}/fe/__init__.py (100%) rename {src => codewiki/src}/fe/background_worker.py (95%) rename {src => codewiki/src}/fe/cache_manager.py (98%) rename {src => codewiki/src}/fe/config.py (100%) rename {src => codewiki/src}/fe/github_processor.py (100%) rename {src => codewiki/src}/fe/models.py (100%) rename {src => codewiki/src}/fe/routes.py (99%) rename {src => codewiki/src}/fe/template_utils.py (100%) rename {src => codewiki/src}/fe/templates.py (100%) rename {src => codewiki/src}/fe/visualise_docs.py (99%) rename {src => codewiki/src}/fe/web_app.py (100%) rename {src => codewiki/src}/utils.py (100%) create mode 100644 codewiki/templates/github_pages/.gitkeep create mode 100644 codewiki/templates/github_pages/app.js create mode 100644 codewiki/templates/github_pages/index.html create mode 100644 codewiki/templates/github_pages/styles.css create mode 100644 pyproject.toml delete mode 100644 src/be/dependency_analyzer/__init__.py delete mode 100644 src/be/llm_services.py delete mode 100644 src/config.py diff --git a/.gitignore b/.gitignore index 04a7fff7..b239c5b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,74 @@ config/agent_config.yaml tool/add_header.sh +# Python .venv +venv/ __pycache__/ -*.egg-info +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Tests +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.hypothesis/ +tests/ + +# Jupyter *.ipynb +.ipynb_checkpoints + +# Project specific output/ data/ src/output/ repo_data/ +docs/.cache/ + +# Environment variables .env* *.env **/.env **/.env.* + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Temporary files +*.tmp +*.log +*.bak diff --git a/codewiki/README.md b/codewiki/README.md new file mode 100644 index 00000000..cbf2bc44 --- /dev/null +++ b/codewiki/README.md @@ -0,0 +1,138 @@ +# CodeWiki CLI + +Transform your codebase into comprehensive documentation using AI-powered analysis. + +## Features + +- **Multi-Language Support**: Python, Java, JavaScript, TypeScript, C, C++, C# +- **AI-Powered Analysis**: Uses LLM models for intelligent documentation generation +- **Comprehensive Documentation**: Generates overview, module docs, and architecture diagrams +- **Dependency Analysis**: Tree-sitter based code parsing and dependency graph generation +- **Module Clustering**: Intelligent grouping of related code components +- **GitHub Pages Integration**: Generate beautiful, interactive HTML documentation +- **Git Workflow Support**: Automated branch creation and commit management +- **Secure Configuration**: API keys stored in system keychain +- **Progress Tracking**: Real-time progress updates with ETA +- **Mermaid Diagrams**: Automatic generation of architecture and data flow diagrams + +## Installation + +```bash +pip install codewiki +``` + +## Quick Start + +### 1. Configure CodeWiki + +```bash +codewiki config set \ + --api-key YOUR_API_KEY \ + --base-url https://api.anthropic.com \ + --main-model claude-sonnet-4 \ + --cluster-model claude-sonnet-4 +``` + +### 2. Generate Documentation + +```bash +cd /path/to/your/project +codewiki generate +``` + +Documentation will be created in `./docs/` + +### 3. Generate with GitHub Pages + +```bash +codewiki generate --github-pages +``` + +This creates an interactive HTML viewer at `./docs/index.html` + +## Commands + +### Configuration Management + +```bash +# Set configuration +codewiki config set --api-key --base-url \ + --main-model --cluster-model + +# Show configuration +codewiki config show + +# Validate configuration +codewiki config validate +``` + +### Documentation Generation + +```bash +# Basic generation +codewiki generate + +# Custom output directory +codewiki generate --output ./documentation + +# Create git branch +codewiki generate --create-branch + +# Generate GitHub Pages HTML +codewiki generate --github-pages + +# Full-featured +codewiki generate --create-branch --github-pages --verbose +``` + +## Configuration + +Configuration is stored in: +- API keys: System keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service) +- Settings: `~/.codewiki/config.json` + +## Supported Languages + +| Language | Extensions | +|------------|---------------------| +| Python | `.py` | +| Java | `.java` | +| JavaScript | `.js`, `.jsx` | +| TypeScript | `.ts`, `.tsx` | +| C | `.c`, `.h` | +| C++ | `.cpp`, `.hpp`, etc.| +| C# | `.cs` | + +## Requirements + +- Python 3.12+ +- Git (optional, for branch management) +- LLM API access (Anthropic Claude, OpenAI, etc.) +- Tree-sitter language parsers (automatically installed) +- System keychain support (macOS Keychain, Windows Credential Manager, Linux Secret Service) + +## Development + +### Install from Source + +```bash +git clone https://github.com/yourusername/codewiki.git +cd codewiki +pip install -e . +``` + +### Run Tests + +```bash +pytest tests/ +``` + +## License + +MIT License - see LICENSE file for details + +## Support + +- Documentation: https://github.com/yourusername/codewiki/docs +- Issues: https://github.com/yourusername/codewiki/issues + diff --git a/codewiki/__init__.py b/codewiki/__init__.py new file mode 100644 index 00000000..e2e911d2 --- /dev/null +++ b/codewiki/__init__.py @@ -0,0 +1,14 @@ +""" +CodeWiki: Transform codebases into comprehensive documentation using AI-powered analysis. + +This package provides a CLI tool for generating documentation from code repositories. +""" + +__version__ = "1.0.0" +__author__ = "CodeWiki Contributors" +__license__ = "MIT" + +from codewiki.cli.main import cli + +__all__ = ["cli", "__version__"] + diff --git a/codewiki/__main__.py b/codewiki/__main__.py new file mode 100644 index 00000000..bceeeb9a --- /dev/null +++ b/codewiki/__main__.py @@ -0,0 +1,8 @@ +""" +Entry point for running codewiki as a module: python -m codewiki +""" + +from codewiki.cli.main import cli + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/codewiki/cli/__init__.py b/codewiki/cli/__init__.py new file mode 100644 index 00000000..e6e485a1 --- /dev/null +++ b/codewiki/cli/__init__.py @@ -0,0 +1,4 @@ +"""CLI module for CodeWiki.""" + +__all__ = [] + diff --git a/codewiki/cli/adapters/__init__.py b/codewiki/cli/adapters/__init__.py new file mode 100644 index 00000000..6204b525 --- /dev/null +++ b/codewiki/cli/adapters/__init__.py @@ -0,0 +1,4 @@ +"""Adapters for integrating with backend modules.""" + +__all__ = [] + diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py new file mode 100644 index 00000000..1ed08ae1 --- /dev/null +++ b/codewiki/cli/adapters/doc_generator.py @@ -0,0 +1,245 @@ +""" +CLI adapter for documentation generator backend. + +This adapter wraps the existing backend documentation_generator.py +and provides CLI-specific functionality like progress reporting. +""" + +from pathlib import Path +from typing import Dict, Any +import time +import asyncio +import os + + +from codewiki.cli.utils.progress import ProgressTracker +from codewiki.cli.models.job import DocumentationJob, LLMConfig +from codewiki.cli.utils.errors import APIError + +# Import backend modules +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import Config as BackendConfig, set_cli_context + + +class CLIDocumentationGenerator: + """ + CLI adapter for documentation generation with progress reporting. + + This class wraps the backend documentation generator and adds + CLI-specific features like progress tracking and error handling. + """ + + def __init__( + self, + repo_path: Path, + output_dir: Path, + config: Dict[str, Any], + verbose: bool = False, + generate_html: bool = False + ): + """ + Initialize the CLI documentation generator. + + Args: + repo_path: Repository path + output_dir: Output directory + config: LLM configuration + verbose: Enable verbose output + generate_html: Whether to generate HTML viewer + """ + self.repo_path = repo_path + self.output_dir = output_dir + self.config = config + self.verbose = verbose + self.generate_html = generate_html + self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) + self.job = DocumentationJob() + + # Setup job metadata + self.job.repository_path = str(repo_path) + self.job.repository_name = repo_path.name + self.job.output_directory = str(output_dir) + self.job.llm_config = LLMConfig( + main_model=config.get('main_model', ''), + cluster_model=config.get('cluster_model', ''), + base_url=config.get('base_url', '') + ) + + def generate(self) -> DocumentationJob: + """ + Generate documentation with progress tracking. + + Returns: + Completed DocumentationJob + + Raises: + APIError: If LLM API call fails + """ + self.job.start() + start_time = time.time() + + try: + # Set CLI context for backend + set_cli_context(True) + + # Create backend config with CLI settings + backend_config = BackendConfig.from_cli( + repo_path=str(self.repo_path), + output_dir=str(self.output_dir), + llm_base_url=self.config.get('base_url'), + llm_api_key=self.config.get('api_key'), + main_model=self.config.get('main_model'), + cluster_model=self.config.get('cluster_model') + ) + + # Run backend documentation generation + asyncio.run(self._run_backend_generation(backend_config)) + + # Stage 4: HTML Generation (optional) + if self.generate_html: + self._run_html_generation() + + # Stage 5: Finalization (metadata already created by backend) + self._finalize_job() + + # Complete job + generation_time = time.time() - start_time + self.job.complete() + + return self.job + + except APIError as e: + self.job.fail(str(e)) + raise + except Exception as e: + self.job.fail(str(e)) + raise + + async def _run_backend_generation(self, backend_config: BackendConfig): + """Run the backend documentation generation with progress tracking.""" + + # Stage 1: Dependency Analysis + self.progress_tracker.start_stage(1, "Dependency Analysis") + if self.verbose: + self.progress_tracker.update_stage(0.2, "Initializing dependency analyzer...") + + # Create documentation generator + doc_generator = DocumentationGenerator(backend_config) + + if self.verbose: + self.progress_tracker.update_stage(0.5, "Parsing source files...") + + # Build dependency graph + try: + components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph() + self.job.statistics.total_files_analyzed = len(components) + self.job.statistics.leaf_nodes = len(leaf_nodes) + + if self.verbose: + self.progress_tracker.update_stage(1.0, f"Found {len(leaf_nodes)} leaf nodes") + except Exception as e: + raise APIError(f"Dependency analysis failed: {e}") + + self.progress_tracker.complete_stage() + + # Stage 2: Module Clustering + self.progress_tracker.start_stage(2, "Module Clustering") + if self.verbose: + self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...") + + # Import clustering function + from codewiki.src.be.cluster_modules import cluster_modules + from codewiki.src.utils import file_manager + from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME + + working_dir = str(self.output_dir.absolute()) + file_manager.ensure_directory(working_dir) + first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) + module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) + + try: + if os.path.exists(first_module_tree_path): + module_tree = file_manager.load_json(first_module_tree_path) + else: + module_tree = cluster_modules(leaf_nodes, components, backend_config) + file_manager.save_json(module_tree, first_module_tree_path) + + file_manager.save_json(module_tree, module_tree_path) + self.job.module_count = len(module_tree) + + if self.verbose: + self.progress_tracker.update_stage(1.0, f"Created {len(module_tree)} modules") + except Exception as e: + raise APIError(f"Module clustering failed: {e}") + + self.progress_tracker.complete_stage() + + # Stage 3: Documentation Generation + self.progress_tracker.start_stage(3, "Documentation Generation") + if self.verbose: + self.progress_tracker.update_stage(0.1, "Generating module documentation...") + + try: + # Run the actual documentation generation + await doc_generator.generate_module_documentation(components, leaf_nodes) + + if self.verbose: + self.progress_tracker.update_stage(0.9, "Creating repository overview...") + + # Create metadata + doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) + + # Collect generated files + for file_path in os.listdir(working_dir): + if file_path.endswith('.md') or file_path.endswith('.json'): + self.job.files_generated.append(file_path) + + except Exception as e: + raise APIError(f"Documentation generation failed: {e}") + + self.progress_tracker.complete_stage() + + def _run_html_generation(self): + """Run HTML generation stage.""" + self.progress_tracker.start_stage(4, "HTML Generation") + + from codewiki.cli.html_generator import HTMLGenerator + + # Create module tree placeholder + module_tree = { + "Overview": { + "description": "Repository overview", + "components": [], + "children": {} + } + } + + # Generate HTML + html_generator = HTMLGenerator() + repo_info = html_generator.detect_repository_info(self.repo_path) + + output_path = self.output_dir / "index.html" + html_generator.generate( + output_path=output_path, + title=repo_info['name'], + module_tree=module_tree, + repository_url=repo_info['url'], + github_pages_url=repo_info['github_pages_url'] + ) + + self.job.files_generated.append("index.html") + + if self.verbose: + self.progress_tracker.update_stage(1.0, "Generated index.html") + + self.progress_tracker.complete_stage() + + def _finalize_job(self): + """Finalize the job (metadata already created by backend).""" + # Just verify metadata exists + metadata_path = self.output_dir / "metadata.json" + if not metadata_path.exists(): + # Create our own if backend didn't + with open(metadata_path, 'w') as f: + f.write(self.job.to_json()) + diff --git a/codewiki/cli/commands/__init__.py b/codewiki/cli/commands/__init__.py new file mode 100644 index 00000000..b9cc1b77 --- /dev/null +++ b/codewiki/cli/commands/__init__.py @@ -0,0 +1,4 @@ +"""CLI command implementations.""" + +__all__ = [] + diff --git a/codewiki/cli/commands/config.py b/codewiki/cli/commands/config.py new file mode 100644 index 00000000..87a6873a --- /dev/null +++ b/codewiki/cli/commands/config.py @@ -0,0 +1,381 @@ +""" +Configuration commands for CodeWiki CLI. +""" + +import json +import sys +import click +from typing import Optional + +from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.utils.errors import ( + ConfigurationError, + handle_error, + EXIT_SUCCESS, + EXIT_CONFIG_ERROR +) +from codewiki.cli.utils.validation import ( + validate_url, + validate_api_key, + validate_model_name, + is_top_tier_model, + mask_api_key +) + + +@click.group(name="config") +def config_group(): + """Manage CodeWiki configuration (API credentials and settings).""" + pass + + +@config_group.command(name="set") +@click.option( + "--api-key", + type=str, + help="LLM API key (stored securely in system keychain)" +) +@click.option( + "--base-url", + type=str, + help="LLM API base URL (e.g., https://api.anthropic.com)" +) +@click.option( + "--main-model", + type=str, + help="Primary model for documentation generation" +) +@click.option( + "--cluster-model", + type=str, + help="Model for module clustering (recommend top-tier)" +) +def config_set( + api_key: Optional[str], + base_url: Optional[str], + main_model: Optional[str], + cluster_model: Optional[str] +): + """ + Set configuration values for CodeWiki. + + API keys are stored securely in your system keychain: + • macOS: Keychain Access + • Windows: Credential Manager + • Linux: Secret Service (GNOME Keyring, KWallet) + + Examples: + + \b + # Set all configuration + $ codewiki config set --api-key sk-abc123 --base-url https://api.anthropic.com \\ + --main-model claude-sonnet-4 --cluster-model claude-sonnet-4 + + \b + # Update only API key + $ codewiki config set --api-key sk-new-key + """ + try: + # Check if at least one option is provided + if not any([api_key, base_url, main_model, cluster_model]): + click.echo("No options provided. Use --help for usage information.") + sys.exit(EXIT_CONFIG_ERROR) + + # Validate inputs before saving + validated_data = {} + + if api_key: + validated_data['api_key'] = validate_api_key(api_key) + + if base_url: + validated_data['base_url'] = validate_url(base_url) + + if main_model: + validated_data['main_model'] = validate_model_name(main_model) + + if cluster_model: + validated_data['cluster_model'] = validate_model_name(cluster_model) + + # Create config manager and save + manager = ConfigManager() + manager.load() # Load existing config if present + + manager.save( + api_key=validated_data.get('api_key'), + base_url=validated_data.get('base_url'), + main_model=validated_data.get('main_model'), + cluster_model=validated_data.get('cluster_model') + ) + + # Display success messages + click.echo() + if api_key: + if manager.keyring_available: + click.secho("✓ API key saved to system keychain", fg="green") + else: + click.secho( + "⚠️ System keychain unavailable. API key stored in encrypted file.", + fg="yellow" + ) + + if base_url: + click.secho(f"✓ Base URL: {base_url}", fg="green") + + if main_model: + click.secho(f"✓ Main model: {main_model}", fg="green") + + if cluster_model: + click.secho(f"✓ Cluster model: {cluster_model}", fg="green") + + # Warn if not using top-tier model for clustering + if not is_top_tier_model(cluster_model): + click.secho( + "\n⚠️ Cluster model is not a top-tier LLM. " + "Documentation quality may be suboptimal.", + fg="yellow" + ) + click.echo( + " Recommended models: claude-opus, claude-sonnet-4, gpt-4, gpt-4-turbo" + ) + + click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True)) + + except ConfigurationError as e: + click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) + sys.exit(e.exit_code) + except Exception as e: + sys.exit(handle_error(e)) + + +@config_group.command(name="show") +@click.option( + "--json", + "output_json", + is_flag=True, + help="Output in JSON format" +) +def config_show(output_json: bool): + """ + Display current configuration. + + API keys are masked for security (showing only first and last 4 characters). + + Examples: + + \b + # Display configuration + $ codewiki config show + + \b + # Display as JSON + $ codewiki config show --json + """ + try: + manager = ConfigManager() + + if not manager.load(): + click.secho("\n✗ Configuration not found.", fg="red", err=True) + click.echo("\nPlease run 'codewiki config set' to configure your API credentials:") + click.echo(" codewiki config set --api-key --base-url \\") + click.echo(" --main-model --cluster-model ") + click.echo("\nFor more help: codewiki config set --help") + sys.exit(EXIT_CONFIG_ERROR) + + config = manager.get_config() + api_key = manager.get_api_key() + + if output_json: + # JSON output + output = { + "api_key": mask_api_key(api_key) if api_key else "Not set", + "api_key_storage": "keychain" if manager.keyring_available else "encrypted_file", + "base_url": config.base_url if config else "", + "main_model": config.main_model if config else "", + "cluster_model": config.cluster_model if config else "", + "default_output": config.default_output if config else "docs", + "config_file": str(manager.config_file_path) + } + click.echo(json.dumps(output, indent=2)) + else: + # Human-readable output + click.echo() + click.secho("CodeWiki Configuration", fg="blue", bold=True) + click.echo("━" * 40) + click.echo() + + click.secho("Credentials", fg="cyan", bold=True) + if api_key: + storage = "system keychain" if manager.keyring_available else "encrypted file" + click.echo(f" API Key: {mask_api_key(api_key)} (in {storage})") + else: + click.secho(" API Key: Not set", fg="yellow") + + click.echo() + click.secho("API Settings", fg="cyan", bold=True) + if config: + click.echo(f" Base URL: {config.base_url or 'Not set'}") + click.echo(f" Main Model: {config.main_model or 'Not set'}") + click.echo(f" Cluster Model: {config.cluster_model or 'Not set'}") + else: + click.secho(" Not configured", fg="yellow") + + click.echo() + click.secho("Output Settings", fg="cyan", bold=True) + if config: + click.echo(f" Default Output: {config.default_output}") + + click.echo() + click.echo(f"Configuration file: {manager.config_file_path}") + click.echo() + + except Exception as e: + sys.exit(handle_error(e)) + + +@config_group.command(name="validate") +@click.option( + "--quick", + is_flag=True, + help="Skip API connectivity test" +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed validation steps" +) +def config_validate(quick: bool, verbose: bool): + """ + Validate configuration and test LLM API connectivity. + + Checks: + • Configuration file exists and is valid + • API key is present + • API settings are correctly formatted + • (Optional) API connectivity test + + Examples: + + \b + # Full validation with API test + $ codewiki config validate + + \b + # Quick validation (config only) + $ codewiki config validate --quick + + \b + # Verbose output + $ codewiki config validate --verbose + """ + try: + click.echo() + click.secho("Validating configuration...", fg="blue", bold=True) + click.echo() + + manager = ConfigManager() + + # Step 1: Check config file + if verbose: + click.echo("[1/5] Checking configuration file...") + click.echo(f" Path: {manager.config_file_path}") + + if not manager.load(): + click.secho("✗ Configuration file not found", fg="red") + click.echo() + click.echo("Error: Configuration is incomplete. Run 'codewiki config set --help' for setup instructions.") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(" ✓ File exists", fg="green") + click.secho(" ✓ Valid JSON format", fg="green") + else: + click.secho("✓ Configuration file exists", fg="green") + + # Step 2: Check API key + if verbose: + click.echo() + click.echo("[2/5] Checking API key...") + storage = "system keychain" if manager.keyring_available else "encrypted file" + click.echo(f" Storage: {storage}") + + api_key = manager.get_api_key() + if not api_key: + click.secho("✗ API key missing", fg="red") + click.echo() + click.echo("Error: API key not set. Run 'codewiki config set --api-key '") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(f" ✓ API key retrieved", fg="green") + click.secho(f" ✓ Length: {len(api_key)} characters", fg="green") + else: + click.secho("✓ API key present (stored in keychain)", fg="green") + + # Step 3: Check base URL + config = manager.get_config() + if verbose: + click.echo() + click.echo("[3/5] Checking base URL...") + click.echo(f" URL: {config.base_url}") + + if not config.base_url: + click.secho("✗ Base URL not set", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + try: + validate_url(config.base_url) + if verbose: + click.secho(" ✓ Valid HTTPS URL", fg="green") + else: + click.secho(f"✓ Base URL valid: {config.base_url}", fg="green") + except ConfigurationError as e: + click.secho(f"✗ Invalid base URL: {e.message}", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + # Step 4: Check models + if verbose: + click.echo() + click.echo("[4/5] Checking model configuration...") + click.echo(f" Main model: {config.main_model}") + click.echo(f" Cluster model: {config.cluster_model}") + + if not config.main_model or not config.cluster_model: + click.secho("✗ Models not configured", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(" ✓ Models configured", fg="green") + else: + click.secho(f"✓ Main model configured: {config.main_model}", fg="green") + click.secho(f"✓ Cluster model configured: {config.cluster_model}", fg="green") + + # Warn about non-top-tier cluster model + if not is_top_tier_model(config.cluster_model): + click.secho( + "⚠️ Cluster model is not top-tier. Consider using claude-sonnet-4 or gpt-4.", + fg="yellow" + ) + + # Step 5: API connectivity test (unless --quick) + if not quick: + try: + from openai import OpenAI + client = OpenAI(api_key=api_key, base_url=config.base_url) + response = client.models.list() + click.secho("✓ API connectivity test successful", fg="green") + except Exception as e: + click.secho("✗ API connectivity test failed", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + # Success + click.echo() + click.secho("✓ Configuration is valid!", fg="green", bold=True) + click.echo() + + except ConfigurationError as e: + click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) + sys.exit(e.exit_code) + except Exception as e: + sys.exit(handle_error(e, verbose=verbose)) + diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py new file mode 100644 index 00000000..7a9234ad --- /dev/null +++ b/codewiki/cli/commands/generate.py @@ -0,0 +1,256 @@ +""" +Generate command for documentation generation. +""" + +import sys +from pathlib import Path +from typing import Optional +import click +import time + +from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.utils.errors import ( + ConfigurationError, + RepositoryError, + APIError, + handle_error, + EXIT_SUCCESS, +) +from codewiki.cli.utils.repo_validator import ( + validate_repository, + check_writable_output, + is_git_repository, + get_git_commit_hash, + get_git_branch, +) +from codewiki.cli.utils.logging import create_logger +from codewiki.cli.adapters.doc_generator import CLIDocumentationGenerator +from codewiki.cli.utils.instructions import display_post_generation_instructions +from codewiki.cli.models.job import GenerationOptions + + +@click.command(name="generate") +@click.option( + "--output", + "-o", + type=click.Path(), + default="docs", + help="Output directory for generated documentation (default: ./docs)", +) +@click.option( + "--create-branch", + is_flag=True, + help="Create a new git branch for documentation changes", +) +@click.option( + "--github-pages", + is_flag=True, + help="Generate index.html for GitHub Pages deployment", +) +@click.option( + "--no-cache", + is_flag=True, + help="Force full regeneration, ignoring cache", +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed progress and debug information", +) +@click.pass_context +def generate_command( + ctx, + output: str, + create_branch: bool, + github_pages: bool, + no_cache: bool, + verbose: bool +): + """ + Generate comprehensive documentation for a code repository. + + Analyzes the current repository and generates documentation using LLM-powered + analysis. Documentation is output to ./docs/ by default. + + Examples: + + \b + # Basic generation + $ codewiki generate + + \b + # With git branch creation and GitHub Pages + $ codewiki generate --create-branch --github-pages + + \b + # Force full regeneration + $ codewiki generate --no-cache + """ + logger = create_logger(verbose=verbose) + start_time = time.time() + + try: + # Pre-generation checks + logger.step("Validating configuration...", 1, 4) + + # Load configuration + config_manager = ConfigManager() + if not config_manager.load(): + raise ConfigurationError( + "Configuration not found or invalid.\n\n" + "Please run 'codewiki config set' to configure your LLM API credentials:\n" + " codewiki config set --api-key --base-url \\\n" + " --main-model --cluster-model \n\n" + "For more help: codewiki config --help" + ) + + if not config_manager.is_configured(): + raise ConfigurationError( + "Configuration is incomplete. Please run 'codewiki config validate'" + ) + + config = config_manager.get_config() + api_key = config_manager.get_api_key() + + logger.success("Configuration valid") + + # Validate repository + logger.step("Validating repository...", 2, 4) + + repo_path = Path.cwd() + repo_path, languages = validate_repository(repo_path) + + logger.success(f"Repository valid: {repo_path.name}") + if verbose: + logger.debug(f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}") + + # Check git repository + if not is_git_repository(repo_path): + if create_branch: + raise RepositoryError( + "Not a git repository.\n\n" + "The --create-branch flag requires a git repository.\n\n" + "To initialize a git repository: git init" + ) + else: + logger.warning("Not a git repository. Git features unavailable.") + + # Validate output directory + output_dir = Path(output).expanduser().resolve() + check_writable_output(output_dir.parent) + + logger.success(f"Output directory: {output_dir}") + + # Check for existing documentation + if output_dir.exists() and list(output_dir.glob("*.md")): + if not click.confirm( + f"\n{output_dir} already contains documentation. Overwrite?", + default=True + ): + logger.info("Generation cancelled by user.") + sys.exit(EXIT_SUCCESS) + + # Git branch creation (if requested) + branch_name = None + if create_branch: + logger.step("Creating git branch...", 3, 4) + + from codewiki.cli.git_manager import GitManager + + git_manager = GitManager(repo_path) + + # Check clean working directory + is_clean, status_msg = git_manager.check_clean_working_directory() + if not is_clean: + raise RepositoryError( + "Working directory has uncommitted changes.\n\n" + f"{status_msg}\n\n" + "Cannot create documentation branch with uncommitted changes.\n" + "Please commit or stash your changes first:\n" + " git add -A && git commit -m \"Your message\"\n" + " # or\n" + " git stash" + ) + + # Create branch + branch_name = git_manager.create_documentation_branch() + logger.success(f"Created branch: {branch_name}") + + # Generate documentation + logger.step("Generating documentation...", 4, 4) + click.echo() + + # Create generation options + generation_options = GenerationOptions( + create_branch=create_branch, + github_pages=github_pages, + no_cache=no_cache, + custom_output=output if output != "docs" else None + ) + + # Create generator + generator = CLIDocumentationGenerator( + repo_path=repo_path, + output_dir=output_dir, + config={ + 'main_model': config.main_model, + 'cluster_model': config.cluster_model, + 'base_url': config.base_url, + 'api_key': api_key, + }, + verbose=verbose, + generate_html=github_pages + ) + + # Run generation + job = generator.generate() + + # Post-generation + generation_time = time.time() - start_time + + # Get repository info + repo_url = None + commit_hash = get_git_commit_hash(repo_path) + current_branch = get_git_branch(repo_path) + + if is_git_repository(repo_path): + try: + import git + repo = git.Repo(repo_path) + if repo.remotes: + repo_url = repo.remotes.origin.url + except: + pass + + # Display instructions + display_post_generation_instructions( + output_dir=output_dir, + repo_name=repo_path.name, + repo_url=repo_url, + branch_name=branch_name, + github_pages=github_pages, + files_generated=job.files_generated, + statistics={ + 'module_count': job.module_count, + 'total_files_analyzed': job.statistics.total_files_analyzed, + 'generation_time': generation_time, + 'total_tokens_used': job.statistics.total_tokens_used, + } + ) + + except ConfigurationError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except RepositoryError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except APIError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except KeyboardInterrupt: + click.echo("\n\nInterrupted by user") + sys.exit(130) + except Exception as e: + sys.exit(handle_error(e, verbose=verbose)) + diff --git a/codewiki/cli/config_manager.py b/codewiki/cli/config_manager.py new file mode 100644 index 00000000..b2b333fa --- /dev/null +++ b/codewiki/cli/config_manager.py @@ -0,0 +1,226 @@ +""" +Configuration manager with keyring integration for secure credential storage. +""" + +import json +from pathlib import Path +from typing import Optional +import keyring +from keyring.errors import KeyringError + +from codewiki.cli.models.config import Configuration +from codewiki.cli.utils.errors import ConfigurationError, FileSystemError +from codewiki.cli.utils.fs import ensure_directory, safe_write, safe_read + + +# Keyring configuration +KEYRING_SERVICE = "codewiki" +KEYRING_API_KEY_ACCOUNT = "api_key" + +# Configuration file location +CONFIG_DIR = Path.home() / ".codewiki" +CONFIG_FILE = CONFIG_DIR / "config.json" +CONFIG_VERSION = "1.0" + + +class ConfigManager: + """ + Manages CodeWiki configuration with secure keyring storage for API keys. + + Storage: + - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager, + Linux Secret Service) + - Other settings: ~/.codewiki/config.json + """ + + def __init__(self): + """Initialize the configuration manager.""" + self._api_key: Optional[str] = None + self._config: Optional[Configuration] = None + self._keyring_available = self._check_keyring_available() + + def _check_keyring_available(self) -> bool: + """Check if system keyring is available.""" + try: + # Try to get/set a test value + keyring.get_password(KEYRING_SERVICE, "__test__") + return True + except KeyringError: + return False + + def load(self) -> bool: + """ + Load configuration from file and keyring. + + Returns: + True if configuration exists, False otherwise + """ + # Load from JSON file + if not CONFIG_FILE.exists(): + return False + + try: + content = safe_read(CONFIG_FILE) + data = json.loads(content) + + # Validate version + if data.get('version') != CONFIG_VERSION: + # Could implement migration here + pass + + self._config = Configuration.from_dict(data) + + # Load API key from keyring + try: + self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + except KeyringError: + # Keyring unavailable, API key will be None + pass + + return True + except (json.JSONDecodeError, FileSystemError) as e: + raise ConfigurationError(f"Failed to load configuration: {e}") + + def save( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + main_model: Optional[str] = None, + cluster_model: Optional[str] = None, + default_output: Optional[str] = None + ): + """ + Save configuration to file and keyring. + + Args: + api_key: API key (stored in keyring) + base_url: LLM API base URL + main_model: Primary model + cluster_model: Clustering model + default_output: Default output directory + """ + # Ensure config directory exists + try: + ensure_directory(CONFIG_DIR) + except FileSystemError as e: + raise ConfigurationError(f"Cannot create config directory: {e}") + + # Load existing config or create new + if self._config is None: + if CONFIG_FILE.exists(): + self.load() + else: + self._config = Configuration( + base_url="", + main_model="", + cluster_model="", + default_output="docs" + ) + + # Update fields if provided + if base_url is not None: + self._config.base_url = base_url + if main_model is not None: + self._config.main_model = main_model + if cluster_model is not None: + self._config.cluster_model = cluster_model + if default_output is not None: + self._config.default_output = default_output + + # Validate configuration + self._config.validate() + + # Save API key to keyring + if api_key is not None: + self._api_key = api_key + try: + keyring.set_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT, api_key) + except KeyringError as e: + # Fallback: warn about keyring unavailability + raise ConfigurationError( + f"System keychain unavailable: {e}\n" + f"Please ensure your system keychain is properly configured." + ) + + # Save non-sensitive config to JSON + config_data = { + "version": CONFIG_VERSION, + **self._config.to_dict() + } + + try: + safe_write(CONFIG_FILE, json.dumps(config_data, indent=2)) + except FileSystemError as e: + raise ConfigurationError(f"Failed to save configuration: {e}") + + def get_api_key(self) -> Optional[str]: + """ + Get API key from keyring. + + Returns: + API key or None if not set + """ + if self._api_key is None: + try: + self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + except KeyringError: + pass + + return self._api_key + + def get_config(self) -> Optional[Configuration]: + """ + Get current configuration. + + Returns: + Configuration object or None if not loaded + """ + return self._config + + def is_configured(self) -> bool: + """ + Check if configuration is complete and valid. + + Returns: + True if configured, False otherwise + """ + if self._config is None: + return False + + # Check if API key is set + if self.get_api_key() is None: + return False + + # Check if config is complete + return self._config.is_complete() + + def delete_api_key(self): + """Delete API key from keyring.""" + try: + keyring.delete_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + self._api_key = None + except KeyringError: + pass + + def clear(self): + """Clear all configuration (file and keyring).""" + # Delete API key from keyring + self.delete_api_key() + + # Delete config file + if CONFIG_FILE.exists(): + CONFIG_FILE.unlink() + + self._config = None + self._api_key = None + + @property + def keyring_available(self) -> bool: + """Check if keyring is available.""" + return self._keyring_available + + @property + def config_file_path(self) -> Path: + """Get configuration file path.""" + return CONFIG_FILE + diff --git a/codewiki/cli/git_manager.py b/codewiki/cli/git_manager.py new file mode 100644 index 00000000..e75a9fa0 --- /dev/null +++ b/codewiki/cli/git_manager.py @@ -0,0 +1,227 @@ +""" +Git operations manager for CodeWiki CLI. +""" + +from pathlib import Path +from datetime import datetime +from typing import Optional, Tuple +import git +from git.exc import GitCommandError + +from codewiki.cli.utils.errors import RepositoryError + + +class GitManager: + """ + Manages git operations for documentation generation. + + Handles: + - Status checking + - Branch creation + - Committing documentation + - Remote detection + """ + + def __init__(self, repo_path: Path): + """ + Initialize git manager. + + Args: + repo_path: Path to git repository + + Raises: + RepositoryError: If not a valid git repository + """ + self.repo_path = Path(repo_path).expanduser().resolve() + + try: + self.repo = git.Repo(repo_path, search_parent_directories=True) + except git.InvalidGitRepositoryError: + raise RepositoryError( + f"Not a git repository: {repo_path}\n\n" + "To initialize a git repository: git init" + ) + + def check_clean_working_directory(self) -> Tuple[bool, str]: + """ + Check if working directory is clean (no uncommitted changes). + + Returns: + Tuple of (is_clean, status_message) + """ + if self.repo.is_dirty(untracked_files=True): + status_lines = [] + + # Changed files + changed = [item.a_path for item in self.repo.index.diff(None)] + if changed: + status_lines.append(f"Modified: {', '.join(changed[:3])}") + if len(changed) > 3: + status_lines.append(f"... and {len(changed) - 3} more") + + # Untracked files + untracked = self.repo.untracked_files + if untracked: + status_lines.append(f"Untracked: {', '.join(untracked[:3])}") + if len(untracked) > 3: + status_lines.append(f"... and {len(untracked) - 3} more") + + return False, "\n".join(status_lines) + + return True, "Working directory is clean" + + def create_documentation_branch(self, force: bool = False) -> str: + """ + Create a new documentation branch with timestamp. + + Args: + force: Force creation even if dirty working directory + + Returns: + Branch name + + Raises: + RepositoryError: If working directory is dirty (unless force=True) + """ + # Check working directory + if not force: + is_clean, status_msg = self.check_clean_working_directory() + if not is_clean: + raise RepositoryError( + "Working directory has uncommitted changes.\n\n" + f"{status_msg}\n\n" + "Cannot create documentation branch with uncommitted changes.\n" + "Please commit or stash your changes first:\n" + " git status\n" + " git add -A && git commit -m \"Your message\"\n" + " # or\n" + " git stash\n\n" + "Then re-run: codewiki generate --create-branch" + ) + + # Generate branch name with timestamp + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + branch_name = f"docs/codewiki-{timestamp}" + + # Check if branch already exists (shouldn't happen with timestamp) + existing_branches = [b.name for b in self.repo.branches] + if branch_name in existing_branches: + # Append counter + counter = 1 + while f"{branch_name}-{counter}" in existing_branches: + counter += 1 + branch_name = f"{branch_name}-{counter}" + + try: + # Create and checkout new branch + new_branch = self.repo.create_head(branch_name) + new_branch.checkout() + return branch_name + except GitCommandError as e: + raise RepositoryError(f"Failed to create branch: {e}") + + def commit_documentation( + self, + docs_path: Path, + message: Optional[str] = None + ) -> str: + """ + Commit generated documentation. + + Args: + docs_path: Path to documentation directory + message: Commit message (optional) + + Returns: + Commit hash + + Raises: + RepositoryError: If commit fails + """ + if message is None: + message = "Add generated documentation\n\nGenerated by CodeWiki CLI" + + try: + # Add documentation files + self.repo.index.add([str(docs_path)]) + + # Commit + commit = self.repo.index.commit(message) + + return commit.hexsha + except GitCommandError as e: + raise RepositoryError(f"Failed to commit documentation: {e}") + + def get_remote_url(self, remote_name: str = "origin") -> Optional[str]: + """ + Get remote repository URL. + + Args: + remote_name: Name of remote (default: origin) + + Returns: + Remote URL or None if no remote + """ + try: + remote = self.repo.remote(remote_name) + return remote.url + except ValueError: + return None + + def get_current_branch(self) -> str: + """ + Get current branch name. + + Returns: + Branch name + """ + try: + return self.repo.active_branch.name + except TypeError: + # Detached HEAD + return "HEAD" + + def get_commit_hash(self) -> str: + """ + Get current commit hash. + + Returns: + Commit hash + """ + return self.repo.head.commit.hexsha + + def branch_exists(self, branch_name: str) -> bool: + """ + Check if a branch exists. + + Args: + branch_name: Branch name to check + + Returns: + True if exists, False otherwise + """ + return branch_name in [b.name for b in self.repo.branches] + + def get_github_pr_url(self, branch_name: str) -> Optional[str]: + """ + Get GitHub PR creation URL for a branch. + + Args: + branch_name: Branch name + + Returns: + PR URL or None if not a GitHub repo + """ + remote_url = self.get_remote_url() + if not remote_url or "github.com" not in remote_url: + return None + + # Clean URL + base_url = remote_url.rstrip('/').replace('.git', '') + + # Convert SSH to HTTPS + if base_url.startswith('git@github.com:'): + base_url = base_url.replace('git@github.com:', 'https://github.com/') + + return f"{base_url}/compare/{branch_name}" + diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py new file mode 100644 index 00000000..159cc6bd --- /dev/null +++ b/codewiki/cli/html_generator.py @@ -0,0 +1,177 @@ +""" +HTML generator for GitHub Pages documentation viewer. +""" + +import json +from pathlib import Path +from typing import Optional, Dict, Any + +from codewiki.cli.utils.errors import FileSystemError +from codewiki.cli.utils.fs import safe_write, safe_read + + +class HTMLGenerator: + """ + Generates static HTML documentation viewer for GitHub Pages. + + Creates a self-contained index.html with embedded styles, scripts, + and configuration for client-side markdown rendering. + """ + + def __init__(self, template_dir: Optional[Path] = None): + """ + Initialize HTML generator. + + Args: + template_dir: Path to template directory (default: package templates) + """ + if template_dir is None: + # Use package templates + template_dir = Path(__file__).parent / "templates" / "github_pages" + + self.template_dir = Path(template_dir) + + # Load templates + self.html_template = self._load_template("index.html") + self.styles = self._load_template("styles.css") + self.app_script = self._load_template("app.js") + + def _load_template(self, filename: str) -> str: + """ + Load a template file. + + Args: + filename: Template filename + + Returns: + Template content + + Raises: + FileSystemError: If template cannot be loaded + """ + template_path = self.template_dir / filename + if not template_path.exists(): + raise FileSystemError(f"Template not found: {template_path}") + + return safe_read(template_path) + + def generate( + self, + output_path: Path, + title: str, + module_tree: Dict[str, Any], + repository_url: Optional[str] = None, + github_pages_url: Optional[str] = None, + config: Optional[Dict[str, Any]] = None + ): + """ + Generate HTML documentation viewer. + + Args: + output_path: Output file path (index.html) + title: Documentation title + module_tree: Module tree structure + repository_url: GitHub repository URL + github_pages_url: Expected GitHub Pages URL + config: Additional configuration + """ + # Build configuration + full_config = { + "title": title, + "repository_url": repository_url or "", + "github_pages_url": github_pages_url or "", + "docs_path": ".", + "default_doc": "overview.md", + "navigation": { + "show_file_tree": True, + "collapsible": True, + "default_expanded_depth": 1 + }, + "markdown": { + "syntax_theme": "github", + "line_numbers": True, + "copy_button": True, + "gfm": True + }, + "mermaid": { + "theme": "default", + "flowchart": {"curve": "basis"} + }, + "features": { + "breadcrumbs": True, + "table_of_contents": True, + "search": False + } + } + + if config: + full_config.update(config) + + # Replace template variables + html = self.html_template + html = html.replace("{{TITLE}}", title) + html = html.replace("{{STYLES}}", self.styles) + html = html.replace("{{APP_SCRIPT}}", self.app_script) + html = html.replace("{{CONFIG_JSON}}", json.dumps(full_config, indent=2)) + html = html.replace("{{MODULE_TREE_JSON}}", json.dumps(module_tree, indent=2)) + + # Handle optional repository URL (Mustache-style) + if repository_url: + html = html.replace("{{#REPOSITORY_URL}}", "") + html = html.replace("{{/REPOSITORY_URL}}", "") + html = html.replace("{{REPOSITORY_URL}}", repository_url) + else: + # Remove conditional block + import re + html = re.sub(r'\{\{#REPOSITORY_URL\}\}.*?\{\{/REPOSITORY_URL\}\}', '', html) + + # Write output + safe_write(output_path, html) + + def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]: + """ + Detect repository information from git. + + Args: + repo_path: Repository path + + Returns: + Dictionary with 'name', 'url', 'github_pages_url' + """ + info = { + 'name': repo_path.name, + 'url': None, + 'github_pages_url': None, + } + + try: + import git + repo = git.Repo(repo_path) + + # Get repository name + info['name'] = repo_path.name + + # Get remote URL + if repo.remotes: + remote_url = repo.remotes.origin.url + + # Clean URL + if remote_url.startswith('git@github.com:'): + remote_url = remote_url.replace('git@github.com:', 'https://github.com/') + + remote_url = remote_url.rstrip('/').replace('.git', '') + info['url'] = remote_url + + # Compute GitHub Pages URL + if 'github.com' in remote_url: + parts = remote_url.split('/') + if len(parts) >= 2: + owner = parts[-2] + repo = parts[-1] + info['github_pages_url'] = f"https://{owner}.github.io/{repo}/" + + except Exception: + pass + + return info + diff --git a/codewiki/cli/main.py b/codewiki/cli/main.py new file mode 100644 index 00000000..44b7f751 --- /dev/null +++ b/codewiki/cli/main.py @@ -0,0 +1,56 @@ +""" +Main CLI application for CodeWiki using Click framework. +""" + +import sys +import click +from pathlib import Path + +from codewiki import __version__ + + +@click.group() +@click.version_option(version=__version__, prog_name="CodeWiki CLI") +@click.pass_context +def cli(ctx): + """ + CodeWiki: Transform codebases into comprehensive documentation. + + Generate AI-powered documentation for your code repositories with support + for Python, Java, JavaScript, TypeScript, C, C++, and C#. + """ + # Ensure context object exists + ctx.ensure_object(dict) + + +@cli.command() +def version(): + """Display version information.""" + click.echo(f"CodeWiki CLI v{__version__}") + click.echo("Python-based documentation generator using AI analysis") + + +# Import commands +from codewiki.cli.commands.config import config_group +from codewiki.cli.commands.generate import generate_command + +# Register command groups +cli.add_command(config_group) +cli.add_command(generate_command, name="generate") + + +def main(): + """Entry point for the CLI.""" + try: + cli(obj={}) + except KeyboardInterrupt: + click.echo("\n\nInterrupted by user", err=True) + sys.exit(130) + except Exception as e: + click.secho(f"\n✗ Unexpected error: {e}", fg="red", err=True) + sys.exit(1) + + +if __name__ == "__main__": + main() + diff --git a/codewiki/cli/models/__init__.py b/codewiki/cli/models/__init__.py new file mode 100644 index 00000000..07db84ac --- /dev/null +++ b/codewiki/cli/models/__init__.py @@ -0,0 +1,4 @@ +"""Data models for CLI.""" + +__all__ = [] + diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py new file mode 100644 index 00000000..94ca211b --- /dev/null +++ b/codewiki/cli/models/config.py @@ -0,0 +1,102 @@ +""" +Configuration data models for CodeWiki CLI. + +This module contains the Configuration class which represents persistent +user settings stored in ~/.codewiki/config.json. These settings are converted +to the backend Config class when running documentation generation. +""" + +from dataclasses import dataclass, asdict +from typing import Optional +from pathlib import Path + +from codewiki.cli.utils.validation import ( + validate_url, + validate_api_key, + validate_model_name, +) + + +@dataclass +class Configuration: + """ + CodeWiki configuration data model. + + Attributes: + base_url: LLM API base URL + main_model: Primary model for documentation generation + cluster_model: Model for module clustering + default_output: Default output directory + """ + base_url: str + main_model: str + cluster_model: str + default_output: str = "docs" + + def validate(self): + """ + Validate all configuration fields. + + Raises: + ConfigurationError: If validation fails + """ + validate_url(self.base_url) + validate_model_name(self.main_model) + validate_model_name(self.cluster_model) + + def to_dict(self) -> dict: + """Convert to dictionary.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict) -> 'Configuration': + """ + Create Configuration from dictionary. + + Args: + data: Configuration dictionary + + Returns: + Configuration instance + """ + return cls( + base_url=data.get('base_url', ''), + main_model=data.get('main_model', ''), + cluster_model=data.get('cluster_model', ''), + default_output=data.get('default_output', 'docs'), + ) + + def is_complete(self) -> bool: + """Check if all required fields are set.""" + return bool( + self.base_url and + self.main_model and + self.cluster_model + ) + + def to_backend_config(self, repo_path: str, output_dir: str, api_key: str): + """ + Convert CLI Configuration to Backend Config. + + This method bridges the gap between persistent user settings (CLI Configuration) + and runtime job configuration (Backend Config). + + Args: + repo_path: Path to the repository to document + output_dir: Output directory for generated documentation + api_key: LLM API key (from keyring) + + Returns: + Backend Config instance ready for documentation generation + """ + from codewiki.src.config import Config + + return Config.from_cli( + repo_path=repo_path, + output_dir=output_dir, + llm_base_url=self.base_url, + llm_api_key=api_key, + main_model=self.main_model, + cluster_model=self.cluster_model + ) + diff --git a/codewiki/cli/models/job.py b/codewiki/cli/models/job.py new file mode 100644 index 00000000..c0c49d12 --- /dev/null +++ b/codewiki/cli/models/job.py @@ -0,0 +1,156 @@ +""" +Documentation job data models. +""" + +from dataclasses import dataclass, field, asdict +from datetime import datetime +from typing import List, Optional, Dict, Any +from enum import Enum +import uuid +import json + + +class JobStatus(str, Enum): + """Documentation job status.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class GenerationOptions: + """Options for documentation generation.""" + create_branch: bool = False + github_pages: bool = False + no_cache: bool = False + custom_output: Optional[str] = None + + +@dataclass +class JobStatistics: + """Statistics for a documentation job.""" + total_files_analyzed: int = 0 + leaf_nodes: int = 0 + max_depth: int = 0 + total_tokens_used: int = 0 + + +@dataclass +class LLMConfig: + """LLM configuration for a job.""" + main_model: str + cluster_model: str + base_url: str + + +@dataclass +class DocumentationJob: + """ + Represents a documentation generation job. + + Attributes: + job_id: Unique job identifier + repository_path: Absolute path to repository + repository_name: Repository name + output_directory: Output directory path + commit_hash: Git commit SHA + branch_name: Git branch name (if applicable) + timestamp_start: Job start time + timestamp_end: Job end time (if completed) + status: Current job status + error_message: Error message (if failed) + files_generated: List of generated files + module_count: Number of modules documented + generation_options: Generation options used + llm_config: LLM configuration used + statistics: Job statistics + """ + job_id: str = field(default_factory=lambda: str(uuid.uuid4())) + repository_path: str = "" + repository_name: str = "" + output_directory: str = "" + commit_hash: str = "" + branch_name: Optional[str] = None + timestamp_start: str = field(default_factory=lambda: datetime.now().isoformat()) + timestamp_end: Optional[str] = None + status: JobStatus = JobStatus.PENDING + error_message: Optional[str] = None + files_generated: List[str] = field(default_factory=list) + module_count: int = 0 + generation_options: GenerationOptions = field(default_factory=GenerationOptions) + llm_config: Optional[LLMConfig] = None + statistics: JobStatistics = field(default_factory=JobStatistics) + + def start(self): + """Mark job as started.""" + self.status = JobStatus.RUNNING + self.timestamp_start = datetime.now().isoformat() + + def complete(self): + """Mark job as completed.""" + self.status = JobStatus.COMPLETED + self.timestamp_end = datetime.now().isoformat() + + def fail(self, error_message: str): + """Mark job as failed.""" + self.status = JobStatus.FAILED + self.error_message = error_message + self.timestamp_end = datetime.now().isoformat() + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + data = { + "job_id": self.job_id, + "repository_path": self.repository_path, + "repository_name": self.repository_name, + "output_directory": self.output_directory, + "commit_hash": self.commit_hash, + "branch_name": self.branch_name, + "timestamp_start": self.timestamp_start, + "timestamp_end": self.timestamp_end, + "status": self.status.value if isinstance(self.status, JobStatus) else self.status, + "error_message": self.error_message, + "files_generated": self.files_generated, + "module_count": self.module_count, + "generation_options": asdict(self.generation_options), + "llm_config": asdict(self.llm_config) if self.llm_config else None, + "statistics": asdict(self.statistics), + } + return data + + def to_json(self) -> str: + """Convert to JSON string.""" + return json.dumps(self.to_dict(), indent=2) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'DocumentationJob': + """Create from dictionary.""" + job = cls( + job_id=data.get('job_id', str(uuid.uuid4())), + repository_path=data.get('repository_path', ''), + repository_name=data.get('repository_name', ''), + output_directory=data.get('output_directory', ''), + commit_hash=data.get('commit_hash', ''), + branch_name=data.get('branch_name'), + timestamp_start=data.get('timestamp_start', datetime.now().isoformat()), + timestamp_end=data.get('timestamp_end'), + status=JobStatus(data.get('status', 'pending')), + error_message=data.get('error_message'), + files_generated=data.get('files_generated', []), + module_count=data.get('module_count', 0), + ) + + # Parse nested objects + if 'generation_options' in data: + opts = data['generation_options'] + job.generation_options = GenerationOptions(**opts) + + if 'llm_config' in data and data['llm_config']: + job.llm_config = LLMConfig(**data['llm_config']) + + if 'statistics' in data: + job.statistics = JobStatistics(**data['statistics']) + + return job + diff --git a/codewiki/cli/utils/__init__.py b/codewiki/cli/utils/__init__.py new file mode 100644 index 00000000..a7a69c23 --- /dev/null +++ b/codewiki/cli/utils/__init__.py @@ -0,0 +1,4 @@ +"""Utility functions and helpers for CLI.""" + +__all__ = [] + diff --git a/codewiki/cli/utils/api_errors.py b/codewiki/cli/utils/api_errors.py new file mode 100644 index 00000000..286db4cf --- /dev/null +++ b/codewiki/cli/utils/api_errors.py @@ -0,0 +1,140 @@ +""" +LLM API error handling utilities with fail-fast behavior. +""" + +from typing import Optional +import click + +from codewiki.cli.utils.errors import APIError + + +class APIErrorHandler: + """Handler for LLM API errors with fail-fast behavior.""" + + @staticmethod + def handle_api_error( + error: Exception, + context: Optional[str] = None, + fail_fast: bool = True + ) -> APIError: + """ + Handle LLM API error and convert to APIError. + + Args: + error: The original exception + context: Additional context (e.g., module name) + fail_fast: Whether to fail immediately (default: True) + + Returns: + APIError instance + """ + error_message = str(error) + + # Detect specific error types + if "429" in error_message or "rate limit" in error_message.lower(): + message = ( + "LLM API rate limit exceeded.\n\n" + "The API returned a 429 error, indicating too many requests.\n\n" + "Troubleshooting:\n" + " 1. Wait a few minutes before retrying\n" + " 2. Check your API quota at your provider's dashboard\n" + " 3. Consider upgrading your API plan\n" + " 4. For large repositories, generate during off-peak hours" + ) + elif "401" in error_message or "authentication" in error_message.lower(): + message = ( + "LLM API authentication failed.\n\n" + "Your API key appears to be invalid or expired.\n\n" + "Troubleshooting:\n" + " 1. Verify your API key: codewiki config show\n" + " 2. Update your API key: codewiki config set --api-key \n" + " 3. Check that your API key is active in your provider's dashboard" + ) + elif "timeout" in error_message.lower(): + message = ( + "LLM API request timed out.\n\n" + "The API did not respond within the expected time.\n\n" + "Troubleshooting:\n" + " 1. Check your internet connection\n" + " 2. Verify the API service is operational\n" + " 3. Try again in a few moments\n" + " 4. If the issue persists, contact your API provider" + ) + elif "network" in error_message.lower() or "connection" in error_message.lower(): + message = ( + "Network error while connecting to LLM API.\n\n" + "Could not establish connection to the API.\n\n" + "Troubleshooting:\n" + " 1. Check your internet connection\n" + " 2. Verify the base URL: codewiki config show\n" + " 3. Check if you're behind a proxy or firewall\n" + " 4. Try: curl -I to test connectivity" + ) + else: + message = ( + f"LLM API error: {error_message}\n\n" + "An unexpected error occurred while communicating with the LLM API.\n\n" + "Troubleshooting:\n" + " 1. Check your configuration: codewiki config validate\n" + " 2. Verify API service status\n" + " 3. Review the error message above for specific details" + ) + + if context: + message = f"Context: {context}\n\n{message}" + + return APIError(message) + + @staticmethod + def display_api_error(error: APIError, module_name: Optional[str] = None): + """ + Display API error with formatting. + + Args: + error: The API error + module_name: Optional module name for context + """ + click.echo() + click.secho("✗ LLM API Error", fg="red", bold=True) + click.echo() + + if module_name: + click.echo(f"Module: {module_name}") + click.echo() + + click.echo(error.message) + click.echo() + click.secho( + "Documentation generation stopped. No partial results saved.", + fg="yellow" + ) + click.echo() + + +def wrap_api_call(func, *args, fail_fast: bool = True, context: Optional[str] = None, **kwargs): + """ + Wrap an API call with error handling. + + Args: + func: Function to call + *args: Positional arguments + fail_fast: Whether to raise on error (default: True) + context: Optional context for error message + **kwargs: Keyword arguments + + Returns: + Function result + + Raises: + APIError: If API call fails and fail_fast is True + """ + try: + return func(*args, **kwargs) + except Exception as e: + api_error = APIErrorHandler.handle_api_error(e, context=context, fail_fast=fail_fast) + if fail_fast: + raise api_error + else: + APIErrorHandler.display_api_error(api_error) + return None + diff --git a/codewiki/cli/utils/errors.py b/codewiki/cli/utils/errors.py new file mode 100644 index 00000000..667c4a30 --- /dev/null +++ b/codewiki/cli/utils/errors.py @@ -0,0 +1,113 @@ +""" +Error handling utilities and exit codes for CLI. + +Exit Codes: + 0: Success + 1: General error + 2: Configuration error (missing/invalid credentials) + 3: Repository error (not a git repo, no code files) + 4: LLM API error (including rate limits) + 5: File system error (permissions, disk space) +""" + +import sys +import click +from typing import Optional + + +# Exit codes +EXIT_SUCCESS = 0 +EXIT_GENERAL_ERROR = 1 +EXIT_CONFIG_ERROR = 2 +EXIT_REPOSITORY_ERROR = 3 +EXIT_API_ERROR = 4 +EXIT_FILESYSTEM_ERROR = 5 + + +class CodeWikiError(Exception): + """Base exception for CodeWiki CLI errors.""" + + def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): + self.message = message + self.exit_code = exit_code + super().__init__(self.message) + + +class ConfigurationError(CodeWikiError): + """Configuration-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_CONFIG_ERROR) + + +class RepositoryError(CodeWikiError): + """Repository-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_REPOSITORY_ERROR) + + +class APIError(CodeWikiError): + """LLM API-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_API_ERROR) + + +class FileSystemError(CodeWikiError): + """File system-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_FILESYSTEM_ERROR) + + +def handle_error(error: Exception, verbose: bool = False) -> int: + """ + Handle errors and return appropriate exit code. + + Args: + error: The exception to handle + verbose: Whether to show detailed error information + + Returns: + Exit code for the error + """ + if isinstance(error, CodeWikiError): + click.secho(f"\n✗ Error: {error.message}", fg="red", err=True) + return error.exit_code + else: + click.secho(f"\n✗ Unexpected error: {error}", fg="red", err=True) + if verbose: + import traceback + click.echo(traceback.format_exc(), err=True) + return EXIT_GENERAL_ERROR + + +def error_with_suggestion(message: str, suggestion: str, exit_code: int = EXIT_GENERAL_ERROR): + """ + Display error message with actionable suggestion and exit. + + Args: + message: The error message + suggestion: Suggested action to resolve the error + exit_code: Exit code to use + """ + click.secho(f"\n✗ Error: {message}", fg="red", err=True) + click.echo(f"\n{suggestion}", err=True) + sys.exit(exit_code) + + +def warning(message: str): + """Display a warning message.""" + click.secho(f"⚠️ {message}", fg="yellow") + + +def success(message: str): + """Display a success message.""" + click.secho(f"✓ {message}", fg="green") + + +def info(message: str): + """Display an info message.""" + click.echo(message) + diff --git a/codewiki/cli/utils/fs.py b/codewiki/cli/utils/fs.py new file mode 100644 index 00000000..88f4cb31 --- /dev/null +++ b/codewiki/cli/utils/fs.py @@ -0,0 +1,190 @@ +""" +File system utilities for CLI operations. +""" + +import os +import shutil +from pathlib import Path +from typing import Optional, List + +from codewiki.cli.utils.errors import FileSystemError + + +def ensure_directory(path: Path, mode: int = 0o700) -> Path: + """ + Ensure directory exists, create if necessary. + + Args: + path: Directory path + mode: Directory permissions (default: 0o700 - user only) + + Returns: + Path to the directory + + Raises: + FileSystemError: If directory cannot be created + """ + try: + path = Path(path).expanduser().resolve() + path.mkdir(parents=True, exist_ok=True, mode=mode) + return path + except PermissionError: + raise FileSystemError( + f"Permission denied: Cannot create directory {path}\n" + f"Try: chmod u+w {path.parent}" + ) + except OSError as e: + raise FileSystemError(f"Cannot create directory {path}: {e}") + + +def check_writable(path: Path) -> bool: + """ + Check if a path is writable. + + Args: + path: Path to check + + Returns: + True if writable, False otherwise + """ + path = Path(path).expanduser().resolve() + + if path.exists(): + return os.access(path, os.W_OK) + else: + # Check parent directory if path doesn't exist + parent = path.parent + return parent.exists() and os.access(parent, os.W_OK) + + +def safe_write(path: Path, content: str, encoding: str = "utf-8"): + """ + Safely write content to a file using atomic write (temp file + rename). + + Args: + path: File path + content: Content to write + encoding: File encoding + + Raises: + FileSystemError: If write fails + """ + path = Path(path).expanduser().resolve() + temp_path = path.with_suffix(path.suffix + ".tmp") + + try: + # Write to temp file + with open(temp_path, "w", encoding=encoding) as f: + f.write(content) + + # Atomic rename + temp_path.replace(path) + except Exception as e: + # Clean up temp file if it exists + if temp_path.exists(): + temp_path.unlink() + raise FileSystemError(f"Cannot write to {path}: {e}") + + +def safe_read(path: Path, encoding: str = "utf-8") -> str: + """ + Safely read content from a file. + + Args: + path: File path + encoding: File encoding + + Returns: + File content + + Raises: + FileSystemError: If read fails + """ + path = Path(path).expanduser().resolve() + + try: + with open(path, "r", encoding=encoding) as f: + return f.read() + except FileNotFoundError: + raise FileSystemError(f"File not found: {path}") + except PermissionError: + raise FileSystemError(f"Permission denied: Cannot read {path}") + except Exception as e: + raise FileSystemError(f"Cannot read {path}: {e}") + + +def get_file_size(path: Path) -> int: + """ + Get file size in bytes. + + Args: + path: File path + + Returns: + File size in bytes + """ + return Path(path).stat().st_size + + +def find_files( + directory: Path, + extensions: Optional[List[str]] = None, + recursive: bool = True +) -> List[Path]: + """ + Find files in directory matching extensions. + + Args: + directory: Directory to search + extensions: List of file extensions (e.g., ['.py', '.java']) + recursive: Search recursively + + Returns: + List of matching file paths + """ + directory = Path(directory).expanduser().resolve() + + if not directory.exists(): + return [] + + pattern = "**/*" if recursive else "*" + files = [] + + for path in directory.glob(pattern): + if not path.is_file(): + continue + + if extensions is None or path.suffix in extensions: + files.append(path) + + return files + + +def cleanup_directory(path: Path, keep_hidden: bool = True): + """ + Clean up a directory by removing its contents. + + Args: + path: Directory to clean + keep_hidden: Keep hidden files/directories (starting with .) + + Raises: + FileSystemError: If cleanup fails + """ + path = Path(path).expanduser().resolve() + + if not path.exists(): + return + + try: + for item in path.iterdir(): + if keep_hidden and item.name.startswith('.'): + continue + + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + except Exception as e: + raise FileSystemError(f"Cannot clean directory {path}: {e}") + diff --git a/codewiki/cli/utils/instructions.py b/codewiki/cli/utils/instructions.py new file mode 100644 index 00000000..11085d09 --- /dev/null +++ b/codewiki/cli/utils/instructions.py @@ -0,0 +1,179 @@ +""" +Post-generation instructions generator. +""" + +from pathlib import Path +from typing import Optional +import click + + +def compute_github_pages_url(repo_url: str, repo_name: str) -> str: + """ + Compute expected GitHub Pages URL from repository URL. + + Args: + repo_url: GitHub repository URL + repo_name: Repository name + + Returns: + Expected GitHub Pages URL + """ + # Extract owner from GitHub URL + # e.g., "https://github.com/owner/repo" -> "owner" + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + owner = parts[-2] + repo = parts[-1].replace('.git', '') + return f"https://{owner}.github.io/{repo}/" + + return f"https://YOUR_USERNAME.github.io/{repo_name}/" + + +def get_pr_creation_url(repo_url: str, branch_name: str) -> str: + """ + Get PR creation URL for GitHub. + + Args: + repo_url: GitHub repository URL + branch_name: Branch name + + Returns: + PR creation URL + """ + base_url = repo_url.rstrip('/').replace('.git', '') + return f"{base_url}/compare/{branch_name}" + + +def display_post_generation_instructions( + output_dir: Path, + repo_name: str, + repo_url: Optional[str] = None, + branch_name: Optional[str] = None, + github_pages: bool = False, + files_generated: list = None, + statistics: dict = None +): + """ + Display post-generation instructions. + + Args: + output_dir: Output directory path + repo_name: Repository name + repo_url: GitHub repository URL (optional) + branch_name: Git branch name (optional) + github_pages: Whether GitHub Pages HTML was generated + files_generated: List of generated files + statistics: Generation statistics + """ + click.echo() + click.secho("✓ Documentation generated successfully!", fg="green", bold=True) + click.echo() + + # Output directory + click.secho("Output directory:", fg="cyan", bold=True) + click.echo(f" {output_dir}") + click.echo() + + # Generated files + if files_generated: + click.secho("Generated files:", fg="cyan", bold=True) + for file in files_generated[:10]: # Show first 10 + click.echo(f" - {file}") + if len(files_generated) > 10: + click.echo(f" ... and {len(files_generated) - 10} more") + click.echo() + + # Statistics + if statistics: + click.secho("Statistics:", fg="cyan", bold=True) + if 'module_count' in statistics: + click.echo(f" Total modules: {statistics['module_count']}") + if 'total_files_analyzed' in statistics: + click.echo(f" Files analyzed: {statistics['total_files_analyzed']}") + if 'generation_time' in statistics: + minutes = int(statistics['generation_time'] // 60) + seconds = int(statistics['generation_time'] % 60) + click.echo(f" Generation time: {minutes} minutes {seconds} seconds") + if 'total_tokens_used' in statistics: + tokens = statistics['total_tokens_used'] + click.echo(f" Tokens used: ~{tokens:,}") + click.echo() + + # Next steps + click.secho("Next steps:", fg="cyan", bold=True) + click.echo() + + click.echo("1. Review the generated documentation:") + click.echo(f" cat {output_dir}/overview.md") + if github_pages: + click.echo(f" open {output_dir}/index.html # View in browser") + click.echo() + + if branch_name: + # Git workflow with branch + click.echo("2. Push the documentation branch:") + click.secho(f" git push origin {branch_name}", fg="yellow") + click.echo() + + if repo_url: + pr_url = get_pr_creation_url(repo_url, branch_name) + click.echo("3. Create a Pull Request to merge documentation:") + click.secho(f" {pr_url}", fg="blue") + click.echo() + + click.echo("4. After merge, enable GitHub Pages:") + else: + click.echo("3. Enable GitHub Pages:") + else: + # Direct commit workflow + click.echo("2. Commit the documentation:") + click.secho(" git add docs/", fg="yellow") + click.secho(' git commit -m "Add generated documentation"', fg="yellow") + click.echo() + + click.echo("3. Push to GitHub:") + click.secho(" git push origin main", fg="yellow") + click.echo() + + click.echo("4. Enable GitHub Pages:") + + click.echo(" - Go to repository Settings → Pages") + click.echo(" - Source: Deploy from a branch") + click.echo(" - Branch: main, folder: /docs") + click.echo() + + if repo_url: + github_pages_url = compute_github_pages_url(repo_url, repo_name) + click.echo("5. Your documentation will be available at:") + click.secho(f" {github_pages_url}", fg="blue", bold=True) + click.echo() + + +def display_generation_summary( + success: bool, + error_message: Optional[str] = None, + output_dir: Optional[Path] = None +): + """ + Display generation summary (success or failure). + + Args: + success: Whether generation was successful + error_message: Error message if failed + output_dir: Output directory if successful + """ + if success: + click.echo() + click.secho("✓ Generation completed successfully!", fg="green", bold=True) + if output_dir: + click.echo(f"\nDocumentation saved to: {output_dir}") + click.echo() + else: + click.echo() + click.secho("✗ Generation failed", fg="red", bold=True) + if error_message: + click.echo() + click.echo(error_message) + click.echo() + diff --git a/codewiki/cli/utils/logging.py b/codewiki/cli/utils/logging.py new file mode 100644 index 00000000..80863404 --- /dev/null +++ b/codewiki/cli/utils/logging.py @@ -0,0 +1,85 @@ +""" +Logging utilities for CLI with colored output and progress tracking. +""" + +import sys +from datetime import datetime +from typing import Optional +import click + + +class CLILogger: + """Logger for CLI with support for verbose and normal modes.""" + + def __init__(self, verbose: bool = False): + """ + Initialize the logger. + + Args: + verbose: Enable verbose output + """ + self.verbose = verbose + self.start_time = datetime.now() + + def debug(self, message: str): + """Log debug message (only in verbose mode).""" + if self.verbose: + timestamp = datetime.now().strftime("%H:%M:%S") + click.secho(f"[{timestamp}] {message}", fg="cyan", dim=True) + + def info(self, message: str): + """Log info message.""" + click.echo(message) + + def success(self, message: str): + """Log success message in green.""" + click.secho(f"✓ {message}", fg="green") + + def warning(self, message: str): + """Log warning message in yellow.""" + click.secho(f"⚠️ {message}", fg="yellow") + + def error(self, message: str): + """Log error message in red.""" + click.secho(f"✗ {message}", fg="red", err=True) + + def step(self, message: str, step: Optional[int] = None, total: Optional[int] = None): + """ + Log a processing step. + + Args: + message: Step description + step: Current step number + total: Total number of steps + """ + if step is not None and total is not None: + prefix = f"[{step}/{total}]" + else: + prefix = "→" + + click.secho(f"{prefix} {message}", fg="blue", bold=True) + + def elapsed_time(self) -> str: + """Get elapsed time since logger was created.""" + elapsed = datetime.now() - self.start_time + minutes = int(elapsed.total_seconds() // 60) + seconds = int(elapsed.total_seconds() % 60) + + if minutes > 0: + return f"{minutes}m {seconds}s" + else: + return f"{seconds}s" + + +def create_logger(verbose: bool = False) -> CLILogger: + """ + Create and return a CLI logger. + + Args: + verbose: Enable verbose output + + Returns: + Configured CLILogger instance + """ + return CLILogger(verbose=verbose) + diff --git a/codewiki/cli/utils/progress.py b/codewiki/cli/utils/progress.py new file mode 100644 index 00000000..f61efc18 --- /dev/null +++ b/codewiki/cli/utils/progress.py @@ -0,0 +1,222 @@ +""" +Progress indicator utilities for CLI. +""" + +import time +from typing import Optional, Callable +from datetime import datetime +import click + + +class ProgressTracker: + """ + Progress tracker with stages and ETA estimation. + + Stages: + 1. Dependency Analysis (40% of time) + 2. Module Clustering (20% of time) + 3. Documentation Generation (30% of time) + 4. HTML Generation (5% of time, optional) + 5. Finalization (5% of time) + """ + + # Stage weights (percentage of total time) + STAGE_WEIGHTS = { + 1: 0.40, # Dependency Analysis + 2: 0.20, # Module Clustering + 3: 0.30, # Documentation Generation + 4: 0.05, # HTML Generation (optional) + 5: 0.05, # Finalization + } + + STAGE_NAMES = { + 1: "Dependency Analysis", + 2: "Module Clustering", + 3: "Documentation Generation", + 4: "HTML Generation", + 5: "Finalization", + } + + def __init__(self, total_stages: int = 5, verbose: bool = False): + """ + Initialize progress tracker. + + Args: + total_stages: Number of stages + verbose: Enable verbose output + """ + self.total_stages = total_stages + self.current_stage = 0 + self.stage_progress = 0.0 + self.start_time = time.time() + self.verbose = verbose + self.current_stage_start = self.start_time + + def start_stage(self, stage: int, description: Optional[str] = None): + """ + Start a new stage. + + Args: + stage: Stage number (1-5) + description: Optional custom description + """ + self.current_stage = stage + self.stage_progress = 0.0 + self.current_stage_start = time.time() + + stage_name = description or self.STAGE_NAMES.get(stage, f"Stage {stage}") + + if self.verbose: + elapsed = self._format_elapsed() + click.secho( + f"\n[{elapsed}] Phase {stage}/{self.total_stages}: {stage_name}", + fg="blue", + bold=True + ) + else: + click.secho( + f"[{stage}/{self.total_stages}] {stage_name}", + fg="blue", + bold=True + ) + + def update_stage(self, progress: float, message: Optional[str] = None): + """ + Update progress within current stage. + + Args: + progress: Progress percentage (0.0 to 1.0) + message: Optional progress message + """ + self.stage_progress = min(1.0, max(0.0, progress)) + + if self.verbose and message: + elapsed = self._format_elapsed() + click.echo(f"[{elapsed}] {message}") + + def complete_stage(self, message: Optional[str] = None): + """ + Complete current stage. + + Args: + message: Optional completion message + """ + self.stage_progress = 1.0 + + if self.verbose: + elapsed = self._format_elapsed() + stage_time = time.time() - self.current_stage_start + stage_name = self.STAGE_NAMES.get(self.current_stage, f"Stage {self.current_stage}") + click.secho( + f"[{elapsed}] {stage_name} complete ({stage_time:.1f}s)", + fg="green" + ) + if message: + click.echo(f"[{elapsed}] {message}") + + def get_overall_progress(self) -> float: + """ + Get overall progress percentage. + + Returns: + Progress (0.0 to 1.0) + """ + completed_weight = sum( + self.STAGE_WEIGHTS.get(s, 0) + for s in range(1, self.current_stage) + ) + + current_weight = self.STAGE_WEIGHTS.get(self.current_stage, 0) * self.stage_progress + + return completed_weight + current_weight + + def _format_elapsed(self) -> str: + """Format elapsed time.""" + elapsed = time.time() - self.start_time + minutes = int(elapsed // 60) + seconds = int(elapsed % 60) + + if minutes > 0: + return f"{minutes:02d}:{seconds:02d}" + else: + return f"00:{seconds:02d}" + + def get_eta(self) -> Optional[str]: + """ + Estimate time remaining. + + Returns: + ETA string or None if cannot estimate + """ + elapsed = time.time() - self.start_time + progress = self.get_overall_progress() + + if progress <= 0.0: + return None + + total_estimated = elapsed / progress + remaining = total_estimated - elapsed + + if remaining < 0: + return "< 1 min" + + minutes = int(remaining // 60) + seconds = int(remaining % 60) + + if minutes > 60: + hours = minutes // 60 + minutes = minutes % 60 + return f"{hours}h {minutes}m" + elif minutes > 0: + return f"{minutes}m {seconds}s" + else: + return f"{seconds}s" + + +class ModuleProgressBar: + """Progress bar for module-by-module generation.""" + + def __init__(self, total_modules: int, verbose: bool = False): + """ + Initialize module progress bar. + + Args: + total_modules: Total number of modules to process + verbose: Enable verbose output + """ + self.total_modules = total_modules + self.current_module = 0 + self.verbose = verbose + self.bar = None + + if not verbose: + self.bar = click.progressbar( + length=total_modules, + label="Generating modules", + show_eta=True, + show_percent=True, + ) + self.bar.__enter__() + + def update(self, module_name: str, cached: bool = False): + """ + Update progress for a module. + + Args: + module_name: Name of the module + cached: Whether the module was loaded from cache + """ + self.current_module += 1 + + if self.verbose: + status = "✓ (cached)" if cached else "⟳ (generating)" + click.echo(f" [{self.current_module}/{self.total_modules}] {module_name}... {status}") + elif self.bar: + self.bar.update(1) + + def finish(self): + """Finish progress bar.""" + if self.bar: + self.bar.__exit__(None, None, None) + self.bar = None + diff --git a/codewiki/cli/utils/repo_validator.py b/codewiki/cli/utils/repo_validator.py new file mode 100644 index 00000000..d0b22ef1 --- /dev/null +++ b/codewiki/cli/utils/repo_validator.py @@ -0,0 +1,184 @@ +""" +Repository validation utilities for documentation generation. +""" + +from pathlib import Path +from typing import Tuple, List +import os + +from codewiki.cli.utils.errors import RepositoryError +from codewiki.cli.utils.validation import validate_repository_path, detect_supported_languages + + +# Supported file extensions by language +SUPPORTED_EXTENSIONS = { + '.py', # Python + '.java', # Java + '.js', # JavaScript + '.jsx', # JavaScript (React) + '.ts', # TypeScript + '.tsx', # TypeScript (React) + '.c', # C + '.h', # C headers + '.cpp', # C++ + '.hpp', # C++ headers + '.cc', # C++ + '.hh', # C++ headers + '.cxx', # C++ + '.hxx', # C++ headers + '.cs', # C# +} + + +def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: + """ + Validate repository for documentation generation. + + Checks: + - Path exists and is a directory + - Contains supported code files + - Has sufficient files for meaningful documentation + + Args: + repo_path: Path to repository + + Returns: + Tuple of (validated_path, language_counts) + + Raises: + RepositoryError: If validation fails + """ + # Validate path exists + repo_path = validate_repository_path(repo_path) + + # Detect languages + languages = detect_supported_languages(repo_path) + + if not languages: + raise RepositoryError( + f"No supported code files found in {repo_path}\n\n" + "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#\n\n" + "Please navigate to a code repository or specify a custom directory:\n" + " cd /path/to/your/project\n" + " codewiki generate" + ) + + return repo_path, languages + + +def check_writable_output(output_dir: Path) -> Path: + """ + Check if output directory is writable. + + Args: + output_dir: Output directory path + + Returns: + Validated output directory path + + Raises: + RepositoryError: If output directory is not writable + """ + output_dir = Path(output_dir).expanduser().resolve() + + # Check if output directory exists + if output_dir.exists(): + if not output_dir.is_dir(): + raise RepositoryError( + f"Output path exists but is not a directory: {output_dir}" + ) + + # Check if writable + if not os.access(output_dir, os.W_OK): + raise RepositoryError( + f"Output directory is not writable: {output_dir}\n\n" + f"Try: chmod u+w {output_dir}" + ) + else: + # Check if parent is writable + parent = output_dir.parent + if not parent.exists(): + raise RepositoryError( + f"Parent directory does not exist: {parent}" + ) + + if not os.access(parent, os.W_OK): + raise RepositoryError( + f"Cannot create output directory (parent not writable): {parent}\n\n" + f"Try: chmod u+w {parent}" + ) + + return output_dir + + +def is_git_repository(repo_path: Path) -> bool: + """ + Check if path is a git repository. + + Args: + repo_path: Path to check + + Returns: + True if git repository, False otherwise + """ + git_dir = repo_path / ".git" + return git_dir.exists() and git_dir.is_dir() + + +def get_git_commit_hash(repo_path: Path) -> str: + """ + Get current git commit hash. + + Args: + repo_path: Repository path + + Returns: + Commit hash or empty string if not a git repo + """ + if not is_git_repository(repo_path): + return "" + + try: + import git + repo = git.Repo(repo_path) + return repo.head.commit.hexsha + except Exception: + return "" + + +def get_git_branch(repo_path: Path) -> str: + """ + Get current git branch name. + + Args: + repo_path: Repository path + + Returns: + Branch name or empty string if not a git repo + """ + if not is_git_repository(repo_path): + return "" + + try: + import git + repo = git.Repo(repo_path) + return repo.active_branch.name + except Exception: + return "" + + +def count_code_files(repo_path: Path) -> int: + """ + Count supported code files in repository. + + Args: + repo_path: Repository path + + Returns: + Number of code files + """ + count = 0 + for ext in SUPPORTED_EXTENSIONS: + count += len(list(repo_path.rglob(f"*{ext}"))) + return count + diff --git a/codewiki/cli/utils/validation.py b/codewiki/cli/utils/validation.py new file mode 100644 index 00000000..da76500d --- /dev/null +++ b/codewiki/cli/utils/validation.py @@ -0,0 +1,231 @@ +""" +Validation utilities for CLI inputs and configuration. +""" + +import re +from pathlib import Path +from typing import Optional, List, Tuple +from urllib.parse import urlparse + +from codewiki.cli.utils.errors import ConfigurationError, RepositoryError + + +def validate_url(url: str, require_https: bool = True, allow_localhost: bool = True) -> str: + """ + Validate URL format. + + Args: + url: URL to validate + require_https: Require HTTPS scheme (except localhost) + allow_localhost: Allow localhost URLs + + Returns: + Validated URL + + Raises: + ConfigurationError: If URL is invalid + """ + try: + parsed = urlparse(url) + + # Check scheme + if not parsed.scheme: + raise ConfigurationError(f"Invalid URL (missing scheme): {url}") + + # Check HTTPS requirement + if require_https and parsed.scheme != 'https': + # Allow HTTP for localhost + if allow_localhost and parsed.hostname in ['localhost', '127.0.0.1', '::1']: + pass + else: + raise ConfigurationError( + f"URL must use HTTPS: {url}\n" + f"HTTP is only allowed for localhost" + ) + + # Check hostname + if not parsed.hostname: + raise ConfigurationError(f"Invalid URL (missing hostname): {url}") + + return url + except ValueError as e: + raise ConfigurationError(f"Invalid URL format: {url}\nError: {e}") + + +def validate_api_key(api_key: str, min_length: int = 10) -> str: + """ + Validate API key format. + + Args: + api_key: API key to validate + min_length: Minimum key length + + Returns: + Validated API key + + Raises: + ConfigurationError: If API key is invalid + """ + if not api_key or not api_key.strip(): + raise ConfigurationError("API key cannot be empty") + + api_key = api_key.strip() + + if len(api_key) < min_length: + raise ConfigurationError( + f"API key too short (minimum {min_length} characters)" + ) + + return api_key + + +def validate_model_name(model: str) -> str: + """ + Validate model name format. + + Args: + model: Model name to validate + + Returns: + Validated model name + + Raises: + ConfigurationError: If model name is invalid + """ + if not model or not model.strip(): + raise ConfigurationError("Model name cannot be empty") + + return model.strip() + + +def validate_output_directory(path: str) -> Path: + """ + Validate output directory path. + + Args: + path: Directory path to validate + + Returns: + Validated Path object + + Raises: + ConfigurationError: If path is invalid + """ + if not path or not path.strip(): + raise ConfigurationError("Output directory cannot be empty") + + try: + resolved_path = Path(path).expanduser().resolve() + + # Check if path is writable (or parent is writable if path doesn't exist) + if resolved_path.exists(): + if not resolved_path.is_dir(): + raise ConfigurationError( + f"Output path exists but is not a directory: {path}" + ) + + return resolved_path + except Exception as e: + raise ConfigurationError(f"Invalid output directory path: {path}\nError: {e}") + + +def validate_repository_path(path: Path) -> Path: + """ + Validate repository path exists and contains code files. + + Args: + path: Repository path to validate + + Returns: + Validated Path object + + Raises: + RepositoryError: If repository is invalid + """ + path = Path(path).expanduser().resolve() + + if not path.exists(): + raise RepositoryError(f"Repository path does not exist: {path}") + + if not path.is_dir(): + raise RepositoryError(f"Repository path is not a directory: {path}") + + return path + + +def detect_supported_languages(directory: Path) -> List[Tuple[str, int]]: + """ + Detect supported programming languages in a directory. + + Args: + directory: Directory to scan + + Returns: + List of (language, file_count) tuples + """ + language_extensions = { + 'Python': ['.py'], + 'Java': ['.java'], + 'JavaScript': ['.js', '.jsx'], + 'TypeScript': ['.ts', '.tsx'], + 'C': ['.c', '.h'], + 'C++': ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], + 'C#': ['.cs'], + } + + language_counts = {} + + for language, extensions in language_extensions.items(): + count = 0 + for ext in extensions: + count += len(list(directory.rglob(f"*{ext}"))) + + if count > 0: + language_counts[language] = count + + # Sort by count descending + return sorted(language_counts.items(), key=lambda x: x[1], reverse=True) + + +def is_top_tier_model(model: str) -> bool: + """ + Check if a model is considered top-tier for clustering. + + Args: + model: Model name + + Returns: + True if top-tier, False otherwise + """ + top_tier_models = [ + 'claude-opus', + 'claude-sonnet-4', + 'gpt-4', + 'gpt-4-turbo', + 'gemini-1.5-pro', + ] + + model_lower = model.lower() + return any(tier in model_lower for tier in top_tier_models) + + +def mask_api_key(api_key: str, visible_chars: int = 4) -> str: + """ + Mask API key for display, showing only first and last few characters. + + Args: + api_key: API key to mask + visible_chars: Number of visible characters at start and end + + Returns: + Masked API key (e.g., "sk-1234...5678") + """ + if not api_key: + return "Not set" + + if len(api_key) <= visible_chars * 2: + # Key too short, mask everything except edges + return f"{api_key[:2]}...{api_key[-2:]}" + + return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" + diff --git a/codewiki/py.typed b/codewiki/py.typed new file mode 100644 index 00000000..484056b3 --- /dev/null +++ b/codewiki/py.typed @@ -0,0 +1,2 @@ +# PEP 561 marker file for type checking support + diff --git a/run_web_app.py b/codewiki/run_web_app.py similarity index 100% rename from run_web_app.py rename to codewiki/run_web_app.py diff --git a/codewiki/src/__init__.py b/codewiki/src/__init__.py new file mode 100644 index 00000000..9a6e5272 --- /dev/null +++ b/codewiki/src/__init__.py @@ -0,0 +1,2 @@ +"""CodeWiki backend and frontend modules.""" + diff --git a/codewiki/src/be/__init__.py b/codewiki/src/be/__init__.py new file mode 100644 index 00000000..f7696180 --- /dev/null +++ b/codewiki/src/be/__init__.py @@ -0,0 +1,2 @@ +"""CodeWiki backend modules for documentation generation.""" + diff --git a/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py similarity index 67% rename from src/be/agent_orchestrator.py rename to codewiki/src/be/agent_orchestrator.py index 7b7455e3..70fea701 100644 --- a/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -1,5 +1,5 @@ from pydantic_ai import Agent -import logfire +# import logfire import logging import os from typing import Dict, List, Any @@ -8,50 +8,50 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -try: - # Configure logfire with environment variables for Docker compatibility - logfire_token = os.getenv('LOGFIRE_TOKEN') - logfire_project = os.getenv('LOGFIRE_PROJECT_NAME', 'default') - logfire_service = os.getenv('LOGFIRE_SERVICE_NAME', 'default') +# try: +# # Configure logfire with environment variables for Docker compatibility +# logfire_token = os.getenv('LOGFIRE_TOKEN') +# logfire_project = os.getenv('LOGFIRE_PROJECT_NAME', 'default') +# logfire_service = os.getenv('LOGFIRE_SERVICE_NAME', 'default') - if logfire_token: - # Configure with explicit token (for Docker) - logfire.configure( - token=logfire_token, - project_name=logfire_project, - service_name=logfire_service, - ) - else: - # Use default configuration (for local development with logfire auth) - logfire.configure( - project_name=logfire_project, - service_name=logfire_service, - ) +# if logfire_token: +# # Configure with explicit token (for Docker) +# logfire.configure( +# token=logfire_token, +# project_name=logfire_project, +# service_name=logfire_service, +# ) +# else: +# # Use default configuration (for local development with logfire auth) +# logfire.configure( +# project_name=logfire_project, +# service_name=logfire_service, +# ) - logfire.instrument_pydantic_ai() - logger.info(f"Logfire configured successfully for project: {logfire_project}") +# logfire.instrument_pydantic_ai() +# logger.info(f"Logfire configured successfully for project: {logfire_project}") -except Exception as e: - logger.warning(f"Failed to configure logfire: {e}") +# except Exception as e: +# logger.warning(f"Failed to configure logfire: {e}") # Local imports -from .agent_tools.deps import CodeWikiDeps -from .agent_tools.read_code_components import read_code_components_tool -from .agent_tools.str_replace_editor import str_replace_editor_tool -from .agent_tools.generate_sub_module_documentations import generate_sub_module_documentation_tool -from .llm_services import fallback_models -from .prompt_template import ( +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool +from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool +from codewiki.src.be.agent_tools.generate_sub_module_documentations import generate_sub_module_documentation_tool +from codewiki.src.be.llm_services import create_fallback_models +from codewiki.src.be.prompt_template import ( SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt, ) from .utils import is_complex_module -from config import ( +from codewiki.src.config import ( Config, MODULE_TREE_FILENAME, ) -from utils import file_manager -from .dependency_analyzer.models.core import Node +from codewiki.src.utils import file_manager +from codewiki.src.be.dependency_analyzer.models.core import Node class AgentOrchestrator: @@ -59,13 +59,14 @@ class AgentOrchestrator: def __init__(self, config: Config): self.config = config + self.fallback_models = create_fallback_models(config) def create_agent(self, module_name: str, components: Dict[str, Any], core_component_ids: List[str]) -> Agent: """Create an appropriate agent based on module complexity.""" if is_complex_module(components, core_component_ids): return Agent( - fallback_models, + self.fallback_models, name=module_name, deps_type=CodeWikiDeps, tools=[ @@ -77,7 +78,7 @@ def create_agent(self, module_name: str, components: Dict[str, Any], ) else: return Agent( - fallback_models, + self.fallback_models, name=module_name, deps_type=CodeWikiDeps, tools=[read_code_components_tool, str_replace_editor_tool], @@ -106,7 +107,8 @@ async def process_module(self, module_name: str, components: Dict[str, Node], current_module_name=module_name, module_tree=module_tree, max_depth=self.config.max_depth, - current_depth=1 + current_depth=1, + config=self.config ) # check if module docs already exists diff --git a/codewiki/src/be/agent_tools/__init__.py b/codewiki/src/be/agent_tools/__init__.py new file mode 100644 index 00000000..b828275e --- /dev/null +++ b/codewiki/src/be/agent_tools/__init__.py @@ -0,0 +1,2 @@ +"""Agent tools for backend processing.""" + diff --git a/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py similarity index 63% rename from src/be/agent_tools/deps.py rename to codewiki/src/be/agent_tools/deps.py index 471a7aa3..e5e6c4ff 100644 --- a/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from ..dependency_analyzer.models.core import Node +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.config import Config @dataclass class CodeWikiDeps: @@ -11,4 +12,5 @@ class CodeWikiDeps: current_module_name: str module_tree: dict[str, any] max_depth: int - current_depth: int \ No newline at end of file + current_depth: int + config: Config # LLM configuration \ No newline at end of file diff --git a/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py similarity index 82% rename from src/be/agent_tools/generate_sub_module_documentations.py rename to codewiki/src/be/agent_tools/generate_sub_module_documentations.py index 4bc275ae..ec1431e0 100644 --- a/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -1,13 +1,13 @@ from pydantic_ai import RunContext, Tool, Agent -from .deps import CodeWikiDeps -from .read_code_components import read_code_components_tool -from .str_replace_editor import str_replace_editor_tool -from ..llm_services import fallback_models -from ..prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt -from ..utils import is_complex_module, count_tokens -from ..cluster_modules import format_potential_core_components -from config import MAX_TOKEN_PER_LEAF_MODULE +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool +from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool +from codewiki.src.be.llm_services import create_fallback_models +from codewiki.src.be.prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt +from codewiki.src.be.utils import is_complex_module, count_tokens +from codewiki.src.be.cluster_modules import format_potential_core_components +from codewiki.src.config import MAX_TOKEN_PER_LEAF_MODULE @@ -23,6 +23,9 @@ async def generate_sub_module_documentation( deps = ctx.deps previous_module_name = deps.current_module_name + + # Create fallback models from config + fallback_models = create_fallback_models(deps.config) # add the sub-module to the module tree value = deps.module_tree diff --git a/src/be/agent_tools/read_code_components.py b/codewiki/src/be/agent_tools/read_code_components.py similarity index 94% rename from src/be/agent_tools/read_code_components.py rename to codewiki/src/be/agent_tools/read_code_components.py index 10005624..0125cbb2 100644 --- a/src/be/agent_tools/read_code_components.py +++ b/codewiki/src/be/agent_tools/read_code_components.py @@ -1,5 +1,5 @@ from pydantic_ai import RunContext, Tool -from .deps import CodeWikiDeps +from codewiki.src.be.agent_tools.deps import CodeWikiDeps async def read_code_components(ctx: RunContext[CodeWikiDeps], component_ids: list[str]) -> str: diff --git a/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py similarity index 100% rename from src/be/agent_tools/str_replace_editor.py rename to codewiki/src/be/agent_tools/str_replace_editor.py diff --git a/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py similarity index 90% rename from src/be/cluster_modules.py rename to codewiki/src/be/cluster_modules.py index a927219c..ec0719d4 100644 --- a/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -3,11 +3,11 @@ import logging logger = logging.getLogger(__name__) -from .dependency_analyzer.models.core import Node -from .llm_services import call_llm -from .utils import count_tokens -from config import MAX_TOKEN_PER_MODULE, CLUSTER_MODEL -from .prompt_template import format_cluster_prompt +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.llm_services import call_llm +from codewiki.src.be.utils import count_tokens +from codewiki.src.config import MAX_TOKEN_PER_MODULE, Config +from codewiki.src.be.prompt_template import format_cluster_prompt def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]: @@ -43,6 +43,7 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str def cluster_modules( leaf_nodes: List[str], components: Dict[str, Node], + config: Config, current_module_tree: dict[str, Any] = {}, current_module_name: str = None, current_module_path: List[str] = [] @@ -57,7 +58,7 @@ def cluster_modules( return {} prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) - response = call_llm(prompt, model=CLUSTER_MODEL) + response = call_llm(prompt, config, model=config.cluster_model) #parse the response try: @@ -104,7 +105,7 @@ def cluster_modules( current_module_path.append(module_name) module_info["children"] = {} - module_info["children"] = cluster_modules(valid_sub_leaf_nodes, components, current_module_tree, module_name, current_module_path) + module_info["children"] = cluster_modules(valid_sub_leaf_nodes, components, config, current_module_tree, module_name, current_module_path) current_module_path.pop() return module_tree \ No newline at end of file diff --git a/codewiki/src/be/dependency_analyzer/__init__.py b/codewiki/src/be/dependency_analyzer/__init__.py new file mode 100644 index 00000000..283d901d --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates +""" +Dependency analyzer module for building and processing import dependency graphs +between Python code components. +""" + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.topo_sort import topological_sort, resolve_cycles, build_graph_from_components, dependency_first_dfs, get_leaf_nodes +from codewiki.src.be.dependency_analyzer.dependency_graphs_builder import DependencyGraphBuilder + +__all__ = [ + 'Node', + 'DependencyParser', + 'topological_sort', + 'resolve_cycles', + 'build_graph_from_components', + 'dependency_first_dfs', + 'get_leaf_nodes', + 'DependencyGraphBuilder' +] \ No newline at end of file diff --git a/src/be/dependency_analyzer/analysis/__init__.py b/codewiki/src/be/dependency_analyzer/analysis/__init__.py similarity index 100% rename from src/be/dependency_analyzer/analysis/__init__.py rename to codewiki/src/be/dependency_analyzer/analysis/__init__.py diff --git a/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py similarity index 96% rename from src/be/dependency_analyzer/analysis/analysis_service.py rename to codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 75d18cb7..003ba0fa 100644 --- a/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -9,12 +9,12 @@ import logging from typing import Dict, List, Optional, Any from pathlib import Path -from ..utils.security import safe_open_text, assert_safe_path -from ..analysis.repo_analyzer import RepoAnalyzer -from ..analysis.call_graph_analyzer import CallGraphAnalyzer -from ..analysis.cloning import clone_repository, cleanup_repository, parse_github_url -from ..models.analysis import AnalysisResult -from ..models.core import Repository +from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text, assert_safe_path +from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer +from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer +from codewiki.src.be.dependency_analyzer.analysis.cloning import clone_repository, cleanup_repository, parse_github_url +from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult +from codewiki.src.be.dependency_analyzer.models.core import Repository logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py similarity index 95% rename from src/be/dependency_analyzer/analysis/call_graph_analyzer.py rename to codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index db71790f..e21eeff9 100644 --- a/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -9,9 +9,9 @@ from typing import Dict, List import logging from pathlib import Path -from ..models.core import Node, CallRelationship -from ..utils.patterns import CODE_EXTENSIONS -from ..utils.security import safe_open_text +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS +from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text logger = logging.getLogger(__name__) @@ -155,7 +155,7 @@ def _analyze_python_file(self, file_path: str, content: str, base_dir: str): content: File content string base_dir: Repository base directory path """ - from ..analyzers.python import analyze_python_file + from codewiki.src.be.dependency_analyzer.analyzers.python import analyze_python_file try: functions, relationships = analyze_python_file( @@ -182,7 +182,7 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): try: logger.debug(f"Starting tree-sitter JavaScript analysis for {file_path}") - from ..analyzers.javascript import analyze_javascript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.javascript import analyze_javascript_file_treesitter functions, relationships = analyze_javascript_file_treesitter( file_path, content, repo_path=repo_dir @@ -212,7 +212,7 @@ def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): try: logger.debug(f"Starting tree-sitter TypeScript analysis for {file_path}") - from ..analyzers.typescript import analyze_typescript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.typescript import analyze_typescript_file_treesitter functions, relationships = analyze_typescript_file_treesitter( file_path, content, repo_path=repo_dir @@ -242,7 +242,7 @@ def _analyze_c_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.c import analyze_c_file + from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file functions, relationships = analyze_c_file(file_path, content, repo_path=repo_dir) @@ -260,7 +260,7 @@ def _analyze_cpp_file(self, file_path: str, content: str, repo_dir: str): file_path: Relative path to the C++ file content: File content string """ - from ..analyzers.cpp import analyze_cpp_file + from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file functions, relationships = analyze_cpp_file( file_path, content, repo_path=repo_dir @@ -281,7 +281,7 @@ def _analyze_java_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.java import analyze_java_file + from codewiki.src.be.dependency_analyzer.analyzers.java import analyze_java_file try: functions, relationships = analyze_java_file(file_path, content, repo_path=repo_dir) @@ -305,7 +305,7 @@ def _analyze_csharp_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.csharp import analyze_csharp_file + from codewiki.src.be.dependency_analyzer.analyzers.csharp import analyze_csharp_file try: functions, relationships = analyze_csharp_file(file_path, content, repo_path=repo_dir) diff --git a/src/be/dependency_analyzer/analysis/cloning.py b/codewiki/src/be/dependency_analyzer/analysis/cloning.py similarity index 100% rename from src/be/dependency_analyzer/analysis/cloning.py rename to codewiki/src/be/dependency_analyzer/analysis/cloning.py diff --git a/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py similarity index 97% rename from src/be/dependency_analyzer/analysis/repo_analyzer.py rename to codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index 37c9eab3..887aaa5d 100644 --- a/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -10,7 +10,7 @@ import json from pathlib import Path from typing import Dict, List, Optional, Union -from ..utils.patterns import DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS +from codewiki.src.be.dependency_analyzer.utils.patterns import DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS class RepoAnalyzer: diff --git a/src/be/dependency_analyzer/analyzers/__init__.py b/codewiki/src/be/dependency_analyzer/analyzers/__init__.py similarity index 100% rename from src/be/dependency_analyzer/analyzers/__init__.py rename to codewiki/src/be/dependency_analyzer/analyzers/__init__.py diff --git a/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py similarity index 98% rename from src/be/dependency_analyzer/analyzers/c.py rename to codewiki/src/be/dependency_analyzer/analyzers/c.py index 778d160d..5d19d5b6 100644 --- a/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_c -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/cpp.py b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/cpp.py rename to codewiki/src/be/dependency_analyzer/analyzers/cpp.py index 3f77dd52..dd89d1b3 100644 --- a/src/be/dependency_analyzer/analyzers/cpp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_cpp -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/csharp.py b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/csharp.py rename to codewiki/src/be/dependency_analyzer/analyzers/csharp.py index 813aff9d..50500aa4 100644 --- a/src/be/dependency_analyzer/analyzers/csharp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_c_sharp -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/java.py b/codewiki/src/be/dependency_analyzer/analyzers/java.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/java.py rename to codewiki/src/be/dependency_analyzer/analyzers/java.py index 8afb0ca7..26f586a1 100644 --- a/src/be/dependency_analyzer/analyzers/java.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/java.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_java -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/javascript.py b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/javascript.py rename to codewiki/src/be/dependency_analyzer/analyzers/javascript.py index 3babe768..2cd9f120 100644 --- a/src/be/dependency_analyzer/analyzers/javascript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py @@ -9,7 +9,7 @@ import tree_sitter_javascript import tree_sitter_typescript -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/python.py b/codewiki/src/be/dependency_analyzer/analyzers/python.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/python.py rename to codewiki/src/be/dependency_analyzer/analyzers/python.py index 12c282ef..c50b1a7e 100644 --- a/src/be/dependency_analyzer/analyzers/python.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/python.py @@ -6,7 +6,7 @@ import os -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/typescript.py b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/typescript.py rename to codewiki/src/be/dependency_analyzer/analyzers/typescript.py index 0a7cc284..24a5b6e0 100644 --- a/src/be/dependency_analyzer/analyzers/typescript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py @@ -9,7 +9,7 @@ from tree_sitter import Parser, Language import tree_sitter_typescript -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py similarity index 70% rename from src/be/dependency_analyzer/ast_parser.py rename to codewiki/src/be/dependency_analyzer/ast_parser.py index 78b2c8a3..98cc9e31 100644 --- a/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -7,14 +7,9 @@ from pathlib import Path import re -from .analysis.analysis_service import AnalysisService -from .utils.patterns import CODE_EXTENSIONS -from .models.core import Node +from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService +from codewiki.src.be.dependency_analyzer.models.core import Node -from pathlib import Path -from config import MAIN_MODEL -from ..llm_services import call_llm -from ..prompt_template import FILTER_FOLDERS_PROMPT logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -148,62 +143,3 @@ def save_dependency_graph(self, output_path: str): logger.info(f"Saved {len(self.components)} components to {output_path}") return result - - def filter_folders(self) -> List[str]: - - def get_items_at_depth_pathlib(project_path, k): - """ - Alternative implementation using pathlib with recursion. - - Args: - project_path (str): Path to the project directory - k (int): Depth level (0 = root level, 1 = one level deep, etc.) - - Returns: - list: List of relative paths (strings) to files and folders at depth k - """ - project_path = Path(project_path).resolve() - - if not project_path.exists(): - return [] - - if k == 0: - return ['.'] - - def get_items_recursive(current_path, current_depth, target_depth): - items = [] - - if current_depth == target_depth: - # We're at the target depth, return this item - rel_path = current_path.relative_to(project_path) - return [str(rel_path)] - - if current_depth < target_depth and current_path.is_dir(): - # Go deeper - try: - for item in current_path.iterdir(): - items.extend(get_items_recursive(item, current_depth + 1, target_depth)) - except PermissionError: - # Skip directories we can't access - pass - - return items - - result = [] - try: - for item in project_path.iterdir(): - result.extend(get_items_recursive(item, 1, k)) - except PermissionError: - pass - - return "\n".join(sorted(result)) - - prompt = FILTER_FOLDERS_PROMPT.format(files=get_items_at_depth_pathlib(self.repo_path, 1), project_name=Path(self.repo_path).name) - - response = call_llm(prompt, model=MAIN_MODEL) - - # regrex get content between [ and ] - match = re.search(r'\[.*?\]', response, re.DOTALL) - - return eval(match.group(0)) - diff --git a/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py similarity index 92% rename from src/be/dependency_analyzer/dependency_graphs_builder.py rename to codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index f0e87017..629e1cdf 100644 --- a/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -1,9 +1,9 @@ from typing import Dict, List, Any import os -from config import Config -from .ast_parser import DependencyParser -from .topo_sort import build_graph_from_components, get_leaf_nodes -from utils import file_manager +from codewiki.src.config import Config +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes +from codewiki.src.utils import file_manager import logging logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/models/__init__.py b/codewiki/src/be/dependency_analyzer/models/__init__.py similarity index 100% rename from src/be/dependency_analyzer/models/__init__.py rename to codewiki/src/be/dependency_analyzer/models/__init__.py diff --git a/src/be/dependency_analyzer/models/analysis.py b/codewiki/src/be/dependency_analyzer/models/analysis.py similarity index 85% rename from src/be/dependency_analyzer/models/analysis.py rename to codewiki/src/be/dependency_analyzer/models/analysis.py index 28e42856..37c0d547 100644 --- a/src/be/dependency_analyzer/models/analysis.py +++ b/codewiki/src/be/dependency_analyzer/models/analysis.py @@ -1,6 +1,6 @@ from pydantic import BaseModel from typing import List, Dict, Any, Optional -from .core import Node, CallRelationship, Repository +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship, Repository class AnalysisResult(BaseModel): diff --git a/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py similarity index 100% rename from src/be/dependency_analyzer/models/core.py rename to codewiki/src/be/dependency_analyzer/models/core.py diff --git a/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py similarity index 99% rename from src/be/dependency_analyzer/topo_sort.py rename to codewiki/src/be/dependency_analyzer/topo_sort.py index 1171464d..57196313 100644 --- a/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -10,7 +10,7 @@ from typing import Dict, List, Set, Any from collections import deque -from .models.core import Node +from codewiki.src.be.dependency_analyzer.models.core import Node logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/utils/__init__.py b/codewiki/src/be/dependency_analyzer/utils/__init__.py similarity index 100% rename from src/be/dependency_analyzer/utils/__init__.py rename to codewiki/src/be/dependency_analyzer/utils/__init__.py diff --git a/src/be/dependency_analyzer/utils/logging_config.py b/codewiki/src/be/dependency_analyzer/utils/logging_config.py similarity index 100% rename from src/be/dependency_analyzer/utils/logging_config.py rename to codewiki/src/be/dependency_analyzer/utils/logging_config.py diff --git a/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py similarity index 100% rename from src/be/dependency_analyzer/utils/patterns.py rename to codewiki/src/be/dependency_analyzer/utils/patterns.py diff --git a/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py similarity index 100% rename from src/be/dependency_analyzer/utils/security.py rename to codewiki/src/be/dependency_analyzer/utils/security.py diff --git a/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py similarity index 95% rename from src/be/documentation_generator.py rename to codewiki/src/be/documentation_generator.py index 35026e81..13eb91d0 100644 --- a/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -10,22 +10,21 @@ logger = logging.getLogger(__name__) # Local imports -from .dependency_analyzer import DependencyGraphBuilder -from .llm_services import call_llm -from .prompt_template import ( +from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder +from codewiki.src.be.llm_services import call_llm +from codewiki.src.be.prompt_template import ( REPO_OVERVIEW_PROMPT, MODULE_OVERVIEW_PROMPT, ) -from .cluster_modules import cluster_modules -from config import ( +from codewiki.src.be.cluster_modules import cluster_modules +from codewiki.src.config import ( Config, FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME, - OVERVIEW_FILENAME, - MAIN_MODEL + OVERVIEW_FILENAME ) -from utils import file_manager -from .agent_orchestrator import AgentOrchestrator +from codewiki.src.utils import file_manager +from codewiki.src.be.agent_orchestrator import AgentOrchestrator class DocumentationGenerator: @@ -44,7 +43,7 @@ def create_documentation_metadata(self, working_dir: str, components: Dict[str, metadata = { "generation_info": { "timestamp": datetime.now().isoformat(), - "main_model": MAIN_MODEL, + "main_model": self.config.main_model, "generator_version": "1.0.0", "repo_path": self.config.repo_path, "commit_id": self.commit_id @@ -216,7 +215,7 @@ async def generate_parent_module_docs(self, module_path: List[str], ) try: - parent_docs = call_llm(prompt) + parent_docs = call_llm(prompt, self.config) # Parse and save parent documentation parent_content = parent_docs.split("")[1].split("")[0].strip() @@ -253,7 +252,7 @@ async def run(self) -> None: module_tree = file_manager.load_json(first_module_tree_path) else: logger.info(f"Module tree not found at {module_tree_path}, clustering modules") - module_tree = cluster_modules(leaf_nodes, components) + module_tree = cluster_modules(leaf_nodes, components, self.config) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) diff --git a/codewiki/src/be/llm_services.py b/codewiki/src/be/llm_services.py new file mode 100644 index 00000000..4ff99dbb --- /dev/null +++ b/codewiki/src/be/llm_services.py @@ -0,0 +1,86 @@ +""" +LLM service factory for creating configured LLM clients. +""" +from pydantic_ai.models.openai import OpenAIModel +from pydantic_ai.providers.openai import OpenAIProvider +from pydantic_ai.models.openai import OpenAIModelSettings +from pydantic_ai.models.fallback import FallbackModel +from openai import OpenAI + +from codewiki.src.config import Config + + +def create_main_model(config: Config) -> OpenAIModel: + """Create the main LLM model from configuration.""" + return OpenAIModel( + model_name=config.main_model, + provider=OpenAIProvider( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ), + settings=OpenAIModelSettings( + temperature=0.0, + max_tokens=32768 + ) + ) + + +def create_fallback_model(config: Config) -> OpenAIModel: + """Create the fallback LLM model from configuration.""" + return OpenAIModel( + model_name=config.fallback_model, + provider=OpenAIProvider( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ), + settings=OpenAIModelSettings( + temperature=0.0, + max_tokens=32768 + ) + ) + + +def create_fallback_models(config: Config) -> FallbackModel: + """Create fallback models chain from configuration.""" + main = create_main_model(config) + fallback = create_fallback_model(config) + return FallbackModel(main, fallback) + + +def create_openai_client(config: Config) -> OpenAI: + """Create OpenAI client from configuration.""" + return OpenAI( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ) + + +def call_llm( + prompt: str, + config: Config, + model: str = None, + temperature: float = 0.0 +) -> str: + """ + Call LLM with the given prompt. + + Args: + prompt: The prompt to send + config: Configuration containing LLM settings + model: Model name (defaults to config.main_model) + temperature: Temperature setting + + Returns: + LLM response text + """ + if model is None: + model = config.main_model + + client = create_openai_client(config) + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature, + max_tokens=32768 + ) + return response.choices[0].message.content \ No newline at end of file diff --git a/src/be/main.py b/codewiki/src/be/main.py similarity index 93% rename from src/be/main.py rename to codewiki/src/be/main.py index 842dd9ad..c84e4e8f 100644 --- a/src/be/main.py +++ b/codewiki/src/be/main.py @@ -17,8 +17,8 @@ logger = logging.getLogger(__name__) # Local imports -from .documentation_generator import DocumentationGenerator -from config import ( +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import ( Config, ) diff --git a/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py similarity index 99% rename from src/be/prompt_template.py rename to codewiki/src/be/prompt_template.py index 72ac3f59..a979c185 100644 --- a/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -209,7 +209,7 @@ """ from typing import Dict, Any -from utils import file_manager +from codewiki.src.utils import file_manager EXTENSION_TO_LANGUAGE = { ".py": "python", diff --git a/src/be/utils.py b/codewiki/src/be/utils.py similarity index 100% rename from src/be/utils.py rename to codewiki/src/be/utils.py diff --git a/codewiki/src/config.py b/codewiki/src/config.py new file mode 100644 index 00000000..4b8f9845 --- /dev/null +++ b/codewiki/src/config.py @@ -0,0 +1,114 @@ +from dataclasses import dataclass +import argparse +import os +import sys +from dotenv import load_dotenv +load_dotenv() + +# Constants +OUTPUT_BASE_DIR = 'output' +DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' +DOCS_DIR = 'docs' +FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' +MODULE_TREE_FILENAME = 'module_tree.json' +OVERVIEW_FILENAME = 'overview.md' +MAX_DEPTH = 2 +MAX_TOKEN_PER_MODULE = 36_369 +MAX_TOKEN_PER_LEAF_MODULE = 16_000 + +# CLI context detection +_CLI_CONTEXT = False + +def set_cli_context(enabled: bool = True): + """Set whether we're running in CLI context (vs web app).""" + global _CLI_CONTEXT + _CLI_CONTEXT = enabled + +def is_cli_context() -> bool: + """Check if running in CLI context.""" + return _CLI_CONTEXT + +# LLM services +# In CLI mode, these will be loaded from ~/.codewiki/config.json + keyring +# In web app mode, use environment variables +MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') +FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') +CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) +LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') +LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') + +@dataclass +class Config: + """Configuration class for CodeWiki.""" + repo_path: str + output_dir: str + dependency_graph_dir: str + docs_dir: str + max_depth: int + # LLM configuration + llm_base_url: str + llm_api_key: str + main_model: str + cluster_model: str + fallback_model: str = FALLBACK_MODEL_1 + + @classmethod + def from_args(cls, args: argparse.Namespace) -> 'Config': + """Create configuration from parsed arguments.""" + repo_name = os.path.basename(os.path.normpath(args.repo_path)) + sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) + + return cls( + repo_path=args.repo_path, + output_dir=OUTPUT_BASE_DIR, + dependency_graph_dir=os.path.join(OUTPUT_BASE_DIR, DEPENDENCY_GRAPHS_DIR), + docs_dir=os.path.join(OUTPUT_BASE_DIR, DOCS_DIR, f"{sanitized_repo_name}-docs"), + max_depth=MAX_DEPTH, + llm_base_url=LLM_BASE_URL, + llm_api_key=LLM_API_KEY, + main_model=MAIN_MODEL, + cluster_model=CLUSTER_MODEL, + fallback_model=FALLBACK_MODEL_1 + ) + + @classmethod + def from_cli( + cls, + repo_path: str, + output_dir: str, + llm_base_url: str, + llm_api_key: str, + main_model: str, + cluster_model: str, + fallback_model: str = FALLBACK_MODEL_1 + ) -> 'Config': + """ + Create configuration for CLI context. + + Args: + repo_path: Repository path + output_dir: Output directory for generated docs + llm_base_url: LLM API base URL + llm_api_key: LLM API key + main_model: Primary model + cluster_model: Clustering model + fallback_model: Fallback model + + Returns: + Config instance + """ + repo_name = os.path.basename(os.path.normpath(repo_path)) + base_output_dir = os.path.join(output_dir, "temp") + + return cls( + repo_path=repo_path, + output_dir=base_output_dir, + dependency_graph_dir=os.path.join(base_output_dir, DEPENDENCY_GRAPHS_DIR), + docs_dir=output_dir, + max_depth=MAX_DEPTH, + llm_base_url=llm_base_url, + llm_api_key=llm_api_key, + main_model=main_model, + cluster_model=cluster_model, + fallback_model=fallback_model + ) \ No newline at end of file diff --git a/src/fe/__init__.py b/codewiki/src/fe/__init__.py similarity index 100% rename from src/fe/__init__.py rename to codewiki/src/fe/__init__.py diff --git a/src/fe/background_worker.py b/codewiki/src/fe/background_worker.py similarity index 95% rename from src/fe/background_worker.py rename to codewiki/src/fe/background_worker.py index bce9a077..eec230f7 100644 --- a/src/fe/background_worker.py +++ b/codewiki/src/fe/background_worker.py @@ -15,13 +15,13 @@ from typing import Dict from dataclasses import asdict -from be.documentation_generator import DocumentationGenerator -from config import Config, MAIN_MODEL +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import Config from .models import JobStatus from .cache_manager import CacheManager from .github_processor import GitHubRepoProcessor from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class BackgroundWorker: """Background worker for processing documentation generation jobs.""" @@ -203,14 +203,12 @@ def _process_job(self, job_id: str): # Generate documentation job.progress = "Analyzing repository structure..." - # Create config for documentation generation - config = Config( - repo_path=temp_repo_dir, - output_dir="output", - dependency_graph_dir=os.path.join("output", "dependency_graphs"), - docs_dir=os.path.join("output", "docs", f"{job_id}-docs"), - max_depth=2 - ) + # Create config for documentation generation (using env vars) + import argparse + args = argparse.Namespace(repo_path=temp_repo_dir) + config = Config.from_args(args) + # Override docs_dir with job-specific directory + config.docs_dir = os.path.join("output", "docs", f"{job_id}-docs") job.progress = "Generating documentation..." diff --git a/src/fe/cache_manager.py b/codewiki/src/fe/cache_manager.py similarity index 98% rename from src/fe/cache_manager.py rename to codewiki/src/fe/cache_manager.py index e9e41d8a..d1560519 100644 --- a/src/fe/cache_manager.py +++ b/codewiki/src/fe/cache_manager.py @@ -10,7 +10,7 @@ from .models import CacheEntry from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class CacheManager: diff --git a/src/fe/config.py b/codewiki/src/fe/config.py similarity index 100% rename from src/fe/config.py rename to codewiki/src/fe/config.py diff --git a/src/fe/github_processor.py b/codewiki/src/fe/github_processor.py similarity index 100% rename from src/fe/github_processor.py rename to codewiki/src/fe/github_processor.py diff --git a/src/fe/models.py b/codewiki/src/fe/models.py similarity index 100% rename from src/fe/models.py rename to codewiki/src/fe/models.py diff --git a/src/fe/routes.py b/codewiki/src/fe/routes.py similarity index 99% rename from src/fe/routes.py rename to codewiki/src/fe/routes.py index 299ec5c9..40da7955 100644 --- a/src/fe/routes.py +++ b/codewiki/src/fe/routes.py @@ -19,7 +19,7 @@ from .templates import WEB_INTERFACE_TEMPLATE from .template_utils import render_template from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class WebRoutes: diff --git a/src/fe/template_utils.py b/codewiki/src/fe/template_utils.py similarity index 100% rename from src/fe/template_utils.py rename to codewiki/src/fe/template_utils.py diff --git a/src/fe/templates.py b/codewiki/src/fe/templates.py similarity index 100% rename from src/fe/templates.py rename to codewiki/src/fe/templates.py diff --git a/src/fe/visualise_docs.py b/codewiki/src/fe/visualise_docs.py similarity index 99% rename from src/fe/visualise_docs.py rename to codewiki/src/fe/visualise_docs.py index bf5a68eb..2c8648dc 100644 --- a/src/fe/visualise_docs.py +++ b/codewiki/src/fe/visualise_docs.py @@ -23,7 +23,7 @@ from .template_utils import render_template from .templates import DOCS_VIEW_TEMPLATE -from utils import file_manager +from codewiki.src.utils import file_manager app = FastAPI(title="Documentation Server", description="Simple documentation server for hosting markdown documentation folders") diff --git a/src/fe/web_app.py b/codewiki/src/fe/web_app.py similarity index 100% rename from src/fe/web_app.py rename to codewiki/src/fe/web_app.py diff --git a/src/utils.py b/codewiki/src/utils.py similarity index 100% rename from src/utils.py rename to codewiki/src/utils.py diff --git a/codewiki/templates/github_pages/.gitkeep b/codewiki/templates/github_pages/.gitkeep new file mode 100644 index 00000000..d5f5da1f --- /dev/null +++ b/codewiki/templates/github_pages/.gitkeep @@ -0,0 +1,3 @@ +# Placeholder file to ensure directory is tracked +# This directory will contain HTML templates for GitHub Pages viewer + diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js new file mode 100644 index 00000000..f75c1eb3 --- /dev/null +++ b/codewiki/templates/github_pages/app.js @@ -0,0 +1,179 @@ +// CodeWiki GitHub Pages Viewer Application + +(function() { + 'use strict'; + + // Initialize marked + marked.setOptions({ + gfm: true, + breaks: true, + highlight: function(code, lang) { + if (lang && hljs.getLanguage(lang)) { + return hljs.highlight(code, { language: lang }).value; + } + return hljs.highlightAuto(code).value; + } + }); + + // Initialize mermaid + mermaid.initialize({ startOnLoad: false, theme: 'default' }); + + // Router + class Router { + constructor() { + this.routes = {}; + this.currentRoute = null; + window.addEventListener('hashchange', () => this.handleRoute()); + } + + handleRoute() { + const hash = window.location.hash.slice(1) || '/'; + const path = hash === '/' ? 'overview.md' : hash; + this.loadMarkdown(path); + this.updateBreadcrumbs(path); + } + + async loadMarkdown(filename) { + try { + const response = await fetch(filename); + if (!response.ok) throw new Error(`Failed to load ${filename}`); + const markdown = await response.text(); + this.renderMarkdown(markdown); + } catch (error) { + console.error('Error loading markdown:', error); + const content = document.getElementById('markdown-content'); + content.innerHTML = `

Error

Could not load ${filename}

`; + } + } + + renderMarkdown(markdown) { + const html = marked.parse(markdown); + const content = document.getElementById('markdown-content'); + content.innerHTML = html; + + // Process mermaid diagrams + const mermaidBlocks = content.querySelectorAll('code.language-mermaid'); + mermaidBlocks.forEach((block, index) => { + const div = document.createElement('div'); + div.className = 'mermaid'; + div.textContent = block.textContent; + block.parentElement.replaceWith(div); + }); + + if (mermaidBlocks.length > 0) { + mermaid.run(); + } + + // Scroll to top + document.getElementById('content').scrollTop = 0; + } + + updateBreadcrumbs(path) { + const breadcrumbs = document.getElementById('breadcrumbs'); + const parts = path.split('/').filter(p => p); + + let html = 'Home'; + let currentPath = ''; + + for (const part of parts) { + currentPath += '/' + part; + html += ' › '; + html += `${part.replace('.md', '')}`; + } + + breadcrumbs.innerHTML = html; + } + } + + // Navigation tree renderer + function renderNavTree(tree, container, depth = 0) { + if (!tree || typeof tree !== 'object') return; + + for (const [moduleName, moduleData] of Object.entries(tree)) { + const item = document.createElement('div'); + item.className = 'nav-item'; + item.style.paddingLeft = `${depth * 1}rem`; + + const link = document.createElement('a'); + link.href = `#/${moduleName}.md`; + link.textContent = moduleName; + link.className = 'nav-link'; + + // Highlight active link + link.addEventListener('click', function() { + document.querySelectorAll('.nav-link').forEach(l => l.classList.remove('active')); + this.classList.add('active'); + }); + + item.appendChild(link); + container.appendChild(item); + + // Render children recursively + if (moduleData.children && Object.keys(moduleData.children).length > 0) { + renderNavTree(moduleData.children, container, depth + 1); + } + } + } + + // Mobile menu toggle + function initMobileMenu() { + const sidebar = document.getElementById('sidebar'); + const toggleBtn = document.getElementById('mobile-menu-toggle'); + + toggleBtn.addEventListener('click', () => { + sidebar.classList.toggle('open'); + }); + + // Close sidebar when clicking outside on mobile + document.addEventListener('click', (e) => { + if (window.innerWidth < 768 && + !sidebar.contains(e.target) && + e.target !== toggleBtn) { + sidebar.classList.remove('open'); + } + }); + } + + // Initialize application + function init() { + // Render navigation tree + const navTree = document.getElementById('nav-tree'); + if (MODULE_TREE) { + // Add overview link first + const overviewItem = document.createElement('div'); + overviewItem.className = 'nav-item'; + const overviewLink = document.createElement('a'); + overviewLink.href = '#/'; + overviewLink.textContent = 'Overview'; + overviewLink.className = 'nav-link active'; + overviewItem.appendChild(overviewLink); + navTree.appendChild(overviewItem); + + // Add divider + const divider = document.createElement('div'); + divider.style.borderTop = '1px solid #e1e4e8'; + divider.style.margin = '0.5rem 0'; + navTree.appendChild(divider); + + // Render module tree + renderNavTree(MODULE_TREE, navTree); + } + + // Initialize router + const router = new Router(); + + // Initialize mobile menu + initMobileMenu(); + + // Initial load + router.handleRoute(); + } + + // Start application when DOM is ready + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } +})(); + diff --git a/codewiki/templates/github_pages/index.html b/codewiki/templates/github_pages/index.html new file mode 100644 index 00000000..f1226dc7 --- /dev/null +++ b/codewiki/templates/github_pages/index.html @@ -0,0 +1,60 @@ + + + + + + {{TITLE}} - Documentation + + + + + + + + + + +
+ + + + +
+ + + + +
+ +
+

Loading...

+
+
+
+
+ + + + + + + + + diff --git a/codewiki/templates/github_pages/styles.css b/codewiki/templates/github_pages/styles.css new file mode 100644 index 00000000..615f630d --- /dev/null +++ b/codewiki/templates/github_pages/styles.css @@ -0,0 +1,244 @@ +/* Reset and Base Styles */ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', + 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', + sans-serif; + line-height: 1.6; + color: #24292e; + background: #ffffff; +} + +#app { + display: flex; + flex-direction: column; + height: 100vh; +} + +/* Header */ +#header { + background: #24292e; + color: #ffffff; + padding: 1rem; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + position: relative; +} + +.header-content { + display: flex; + justify-content: space-between; + align-items: center; +} + +#doc-title { + font-size: 1.5rem; + font-weight: 600; +} + +#header-nav a { + color: #ffffff; + text-decoration: none; + padding: 0.5rem 1rem; + border-radius: 4px; + transition: background 0.2s; +} + +#header-nav a:hover { + background: rgba(255,255,255,0.1); +} + +#mobile-menu-toggle { + display: none; + background: none; + border: none; + color: #ffffff; + font-size: 1.5rem; + cursor: pointer; + padding: 0.5rem; +} + +/* Main Container */ +#main-container { + display: flex; + flex: 1; + overflow: hidden; +} + +/* Sidebar */ +#sidebar { + width: 280px; + background: #f6f8fa; + border-right: 1px solid #e1e4e8; + overflow-y: auto; + padding: 1rem; +} + +.nav-item { + margin: 0.25rem 0; +} + +.nav-link { + display: block; + padding: 0.5rem; + color: #0366d6; + text-decoration: none; + border-radius: 4px; + transition: background 0.2s; +} + +.nav-link:hover { + background: #e1e4e8; +} + +.nav-link.active { + background: #0366d6; + color: #ffffff; +} + +/* Content */ +#content { + flex: 1; + overflow-y: auto; + padding: 2rem; + max-width: 900px; + margin: 0 auto; + width: 100%; +} + +#breadcrumbs { + margin-bottom: 1rem; + color: #586069; + font-size: 0.9rem; +} + +#breadcrumbs a { + color: #0366d6; + text-decoration: none; +} + +#breadcrumbs a:hover { + text-decoration: underline; +} + +.separator { + margin: 0 0.5rem; +} + +/* Markdown Content */ +#markdown-content { + line-height: 1.8; +} + +#markdown-content h1, +#markdown-content h2, +#markdown-content h3 { + margin-top: 1.5rem; + margin-bottom: 1rem; + font-weight: 600; +} + +#markdown-content h1 { + font-size: 2rem; + border-bottom: 1px solid #e1e4e8; + padding-bottom: 0.5rem; +} + +#markdown-content h2 { + font-size: 1.5rem; +} + +#markdown-content h3 { + font-size: 1.25rem; +} + +#markdown-content code { + background: #f6f8fa; + padding: 0.2rem 0.4rem; + border-radius: 3px; + font-family: 'Courier New', monospace; + font-size: 0.9em; +} + +#markdown-content pre { + background: #f6f8fa; + padding: 1rem; + border-radius: 6px; + overflow-x: auto; + margin: 1rem 0; +} + +#markdown-content pre code { + background: none; + padding: 0; +} + +#markdown-content table { + border-collapse: collapse; + width: 100%; + margin: 1rem 0; +} + +#markdown-content th, +#markdown-content td { + border: 1px solid #e1e4e8; + padding: 0.5rem; + text-align: left; +} + +#markdown-content th { + background: #f6f8fa; + font-weight: 600; +} + +#markdown-content a { + color: #0366d6; + text-decoration: none; +} + +#markdown-content a:hover { + text-decoration: underline; +} + +#markdown-content ul, +#markdown-content ol { + margin: 1rem 0; + padding-left: 2rem; +} + +#markdown-content li { + margin: 0.5rem 0; +} + +/* Responsive Design */ +@media (max-width: 768px) { + #sidebar { + position: fixed; + left: -280px; + top: 60px; + height: calc(100vh - 60px); + z-index: 1000; + transition: left 0.3s; + } + + #sidebar.open { + left: 0; + } + + #mobile-menu-toggle { + display: block; + position: absolute; + right: 1rem; + top: 50%; + transform: translateY(-50%); + } + + #content { + padding: 1rem; + } +} + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ce711dc7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,123 @@ +[build-system] +requires = ["setuptools>=68.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "codewiki" +version = "1.0.0" +description = "Transform codebases into comprehensive documentation using AI-powered analysis" +readme = "README.md" +requires-python = ">=3.12" +license = {text = "MIT"} +authors = [ + {name = "CodeWiki Contributors"} +] +keywords = ["documentation", "code-analysis", "ai", "llm", "developer-tools"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Documentation", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +dependencies = [ + "click>=8.1.0", + "keyring>=24.0.0", + "GitPython>=3.1.40", + "Jinja2>=3.1.6", + "tree-sitter>=0.23.2", + "tree-sitter-language-pack>=0.8.0", + "tree-sitter-python>=0.23.6", + "tree-sitter-java>=0.23.5", + "tree-sitter-javascript>=0.21.4", + "tree-sitter-typescript>=0.21.2", + "tree-sitter-c>=0.21.4", + "tree-sitter-cpp>=0.23.4", + "tree-sitter-c-sharp>=0.23.1", + "openai>=1.107.0", + "litellm>=1.77.0", + "pydantic>=2.11.7", + "pydantic-settings>=2.10.1", + "pydantic-ai>=1.0.6", + "requests>=2.32.4", + "python-dotenv>=1.1.1", + "rich>=14.1.0", + "networkx>=3.5", + "psutil>=7.0.0", + "PyYAML>=6.0.2", + "mermaid-parser-py>=0.0.2", + "mermaid-py>=0.8.0" +] + +[external] +# Node.js is required for mermaid-py which validates mermaid diagrams in generated documentation +build-requires = [ + { name = "nodejs", version = ">=14.0.0" } +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-cov>=4.1.0", + "pytest-asyncio>=0.21.0", + "black>=23.0.0", + "mypy>=1.5.0", + "ruff>=0.1.0", +] + +[project.scripts] +codewiki = "codewiki.cli.main:cli" + +[project.urls] +Homepage = "https://github.com/yourusername/codewiki" +Documentation = "https://github.com/yourusername/codewiki/docs" +Repository = "https://github.com/yourusername/codewiki" +Issues = "https://github.com/yourusername/codewiki/issues" + +[tool.setuptools] +packages = [ + "codewiki", + "codewiki.cli", + "codewiki.cli.commands", + "codewiki.cli.models", + "codewiki.cli.utils", + "codewiki.cli.adapters", + "codewiki.src", + "codewiki.src.be", + "codewiki.src.be.agent_tools", + "codewiki.src.be.dependency_analyzer", + "codewiki.src.be.dependency_analyzer.analysis", + "codewiki.src.be.dependency_analyzer.analyzers", + "codewiki.src.be.dependency_analyzer.models", + "codewiki.src.be.dependency_analyzer.utils", + "codewiki.src.fe" +] + +[tool.setuptools.package-data] +codewiki = ["templates/**/*", "py.typed"] + +[tool.black] +line-length = 100 +target-version = ['py312'] + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "-v --cov=codewiki --cov-report=term-missing" + diff --git a/src/be/dependency_analyzer/__init__.py b/src/be/dependency_analyzer/__init__.py deleted file mode 100644 index 3183dac1..00000000 --- a/src/be/dependency_analyzer/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates -""" -Dependency analyzer module for building and processing import dependency graphs -between Python code components. -""" - -from .models.core import Node -from .ast_parser import DependencyParser -from .topo_sort import topological_sort, resolve_cycles, build_graph_from_components, dependency_first_dfs, get_leaf_nodes -from .dependency_graphs_builder import DependencyGraphBuilder - -__all__ = [ - 'Node', - 'DependencyParser', - 'topological_sort', - 'resolve_cycles', - 'build_graph_from_components', - 'dependency_first_dfs', - 'get_leaf_nodes', - 'DependencyGraphBuilder' -] \ No newline at end of file diff --git a/src/be/llm_services.py b/src/be/llm_services.py deleted file mode 100644 index 52b28021..00000000 --- a/src/be/llm_services.py +++ /dev/null @@ -1,50 +0,0 @@ -from pydantic_ai.models.openai import OpenAIModel -from pydantic_ai.providers.openai import OpenAIProvider -from pydantic_ai.models.openai import OpenAIModelSettings -from pydantic_ai.models.fallback import FallbackModel - -from config import MAIN_MODEL, FALLBACK_MODEL_1, LLM_BASE_URL, LLM_API_KEY - - -main_model = OpenAIModel( - model_name=MAIN_MODEL, - provider=OpenAIProvider( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY - ), - settings=OpenAIModelSettings( - temperature=0.0, - max_tokens=32768 - ) -) - -fallback_model_1 = OpenAIModel( - model_name=FALLBACK_MODEL_1, - provider=OpenAIProvider( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY - ), - settings=OpenAIModelSettings( - temperature=0.0, - max_tokens=32768 - ) -) - -fallback_models = FallbackModel(main_model, fallback_model_1) - -# ------------------------------------------------------------ -from openai import OpenAI - -client = OpenAI( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY -) - -def call_llm(prompt: str, model: str = MAIN_MODEL, temperature: float = 0.0): - response = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=temperature, - max_tokens=32768 - ) - return response.choices[0].message.content \ No newline at end of file diff --git a/src/config.py b/src/config.py deleted file mode 100644 index 29157353..00000000 --- a/src/config.py +++ /dev/null @@ -1,46 +0,0 @@ -from dataclasses import dataclass -import argparse -import os -from dotenv import load_dotenv -load_dotenv() - -# Constants -OUTPUT_BASE_DIR = 'output' -DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' -DOCS_DIR = 'docs' -FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' -MODULE_TREE_FILENAME = 'module_tree.json' -OVERVIEW_FILENAME = 'overview.md' -MAX_DEPTH = 2 -MAX_TOKEN_PER_MODULE = 36_369 -MAX_TOKEN_PER_LEAF_MODULE = 16_000 - -# LLM services -MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') -FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') -CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) -LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') -LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') - -@dataclass -class Config: - """Configuration class for CodeWiki.""" - repo_path: str - output_dir: str - dependency_graph_dir: str - docs_dir: str - max_depth: int - - @classmethod - def from_args(cls, args: argparse.Namespace) -> 'Config': - """Create configuration from parsed arguments.""" - repo_name = os.path.basename(os.path.normpath(args.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) - - return cls( - repo_path=args.repo_path, - output_dir=OUTPUT_BASE_DIR, - dependency_graph_dir=os.path.join(OUTPUT_BASE_DIR, DEPENDENCY_GRAPHS_DIR), - docs_dir=os.path.join(OUTPUT_BASE_DIR, DOCS_DIR, f"{sanitized_repo_name}-docs"), - max_depth=MAX_DEPTH - ) \ No newline at end of file From 568c38bfd7f998ee1cbc40f17b9acf7bf95b1cca Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:18:50 +0700 Subject: [PATCH 02/17] adjust logs --- codewiki/src/be/agent_orchestrator.py | 15 ++++-- codewiki/src/be/cluster_modules.py | 4 +- .../analysis/analysis_service.py | 46 +++++++++---------- .../src/be/dependency_analyzer/ast_parser.py | 6 +-- .../dependency_graphs_builder.py | 2 +- .../src/be/dependency_analyzer/topo_sort.py | 2 +- codewiki/src/be/documentation_generator.py | 37 ++++++++------- codewiki/src/be/main.py | 2 +- codewiki/src/be/utils.py | 8 ++-- 9 files changed, 67 insertions(+), 55 deletions(-) diff --git a/codewiki/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py index 70fea701..78ea3ac2 100644 --- a/codewiki/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -29,7 +29,7 @@ # ) # logfire.instrument_pydantic_ai() -# logger.info(f"Logfire configured successfully for project: {logfire_project}") +# logger.debug(f"Logfire configured successfully for project: {logfire_project}") # except Exception as e: # logger.warning(f"Failed to configure logfire: {e}") @@ -45,10 +45,11 @@ LEAF_SYSTEM_PROMPT, format_user_prompt, ) -from .utils import is_complex_module +from codewiki.src.be.utils import is_complex_module from codewiki.src.config import ( Config, MODULE_TREE_FILENAME, + OVERVIEW_FILENAME, ) from codewiki.src.utils import file_manager from codewiki.src.be.dependency_analyzer.models.core import Node @@ -88,7 +89,7 @@ def create_agent(self, module_name: str, components: Dict[str, Any], async def process_module(self, module_name: str, components: Dict[str, Node], core_component_ids: List[str], module_path: List[str], working_dir: str) -> Dict[str, Any]: """Process a single module and generate its documentation.""" - logger.info(f"Processing module: {module_name}") + logger.debug(f"Processing module: {module_name}") # Load or create module tree module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) @@ -111,6 +112,12 @@ async def process_module(self, module_name: str, components: Dict[str, Node], config=self.config ) + # check if overview docs already exists + overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) + if os.path.exists(overview_docs_path): + logger.info(f"Overview docs already exists at {overview_docs_path}") + return module_tree + # check if module docs already exists docs_path = os.path.join(working_dir, f"{module_name}.md") if os.path.exists(docs_path): @@ -131,7 +138,7 @@ async def process_module(self, module_name: str, components: Dict[str, Node], # Save updated module tree file_manager.save_json(deps.module_tree, module_tree_path) - logger.info(f"Successfully processed module: {module_name}") + logger.debug(f"Successfully processed module: {module_name}") return deps.module_tree diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py index ec0719d4..69eebf79 100644 --- a/codewiki/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -54,7 +54,7 @@ def cluster_modules( potential_core_components, potential_core_components_with_code = format_potential_core_components(leaf_nodes, components) if count_tokens(potential_core_components_with_code) <= MAX_TOKEN_PER_MODULE: - logger.info(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens") + logger.debug(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens") return {} prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) @@ -79,7 +79,7 @@ def cluster_modules( # check if the module tree is valid if len(module_tree) <= 1: - logger.info(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules") + logger.debug(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules") return {} if current_module_tree == {}: diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 003ba0fa..eb9d27b5 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -55,7 +55,7 @@ def analyze_local_repository( Dict with analysis results including nodes and relationships """ try: - logger.info(f"Analyzing local repository at {repo_path}") + logger.debug(f"Analyzing local repository at {repo_path}") # Get repo analyzer to find files repo_analyzer = RepoAnalyzer() @@ -71,9 +71,9 @@ def analyze_local_repository( # Limit number of files if len(code_files) > max_files: code_files = code_files[:max_files] - logger.info(f"Limited analysis to {max_files} files") + logger.debug(f"Limited analysis to {max_files} files") - logger.info(f"Analyzing {len(code_files)} files") + logger.debug(f"Analyzing {len(code_files)} files") # Analyze files result = self.call_graph_analyzer.analyze_code_files(code_files, repo_path) @@ -115,18 +115,18 @@ def analyze_repository_full( """ temp_dir = None try: - logger.info(f"Starting full analysis of {github_url}") + logger.debug(f"Starting full analysis of {github_url}") temp_dir = self._clone_repository(github_url) repo_info = self._parse_repository_info(github_url) - logger.info("Analyzing repository file structure...") + logger.debug("Analyzing repository file structure...") structure_result = self._analyze_structure(temp_dir, include_patterns, exclude_patterns) - logger.info(f"Found {structure_result['summary']['total_files']} files to analyze.") + logger.debug(f"Found {structure_result['summary']['total_files']} files to analyze.") - logger.info("Starting call graph analysis...") + logger.debug("Starting call graph analysis...") call_graph_result = self._analyze_call_graph(structure_result["file_tree"], temp_dir) - logger.info( + logger.debug( f"Call graph analysis complete. Found {call_graph_result['call_graph']['total_functions']} functions." ) @@ -152,10 +152,10 @@ def analyze_repository_full( readme_content=readme_content, ) - logger.info(f"Cleaning up temporary repository directory: {temp_dir}") + logger.debug(f"Cleaning up temporary repository directory: {temp_dir}") self._cleanup_repository(temp_dir) - logger.info( + logger.debug( f"Analysis completed: {analysis_result.summary['total_functions']} functions found" ) return analysis_result @@ -185,7 +185,7 @@ def analyze_repository_structure_only( """ temp_dir = None try: - logger.info(f"Starting structure analysis of {github_url}") + logger.debug(f"Starting structure analysis of {github_url}") temp_dir = self._clone_repository(github_url) repo_info = self._parse_repository_info(github_url) @@ -203,7 +203,7 @@ def analyze_repository_structure_only( self._cleanup_repository(temp_dir) - logger.info( + logger.debug( f"Structure analysis completed: {result['file_summary']['total_files']} files found" ) return result @@ -216,9 +216,9 @@ def analyze_repository_structure_only( def _clone_repository(self, github_url: str) -> str: """Clone repository and return temp dir path.""" - logger.info(f"Cloning {github_url}...") + logger.debug(f"Cloning {github_url}...") temp_dir = clone_repository(github_url) - logger.info(f"Repository cloned to {temp_dir}") + logger.debug(f"Repository cloned to {temp_dir}") self._temp_directories.append(temp_dir) return temp_dir @@ -233,7 +233,7 @@ def _analyze_structure( exclude_patterns: Optional[List[str]], ) -> Dict[str, Any]: """Analyze repository file structure with filtering.""" - logger.info( + logger.debug( f"Initializing RepoAnalyzer with include: {include_patterns}, exclude: {exclude_patterns}" ) repo_analyzer = RepoAnalyzer(include_patterns, exclude_patterns) @@ -246,12 +246,12 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: # readme_path = Path(repo_dir) / name # if readme_path.exists(): # try: - # logger.info(f"Found README file at {readme_path}") + # logger.debug(f"Found README file at {readme_path}") # return readme_path.read_text(encoding="utf-8") # except Exception as e: # logger.warning(f"Could not read README file at {readme_path}: {e}") # return None - # logger.info("No README file found in repository root.") + # logger.debug("No README file found in repository root.") # return None base = Path(repo_dir) possible_readme_names = ["README.md", "README", "readme.md", "README.txt"] @@ -260,12 +260,12 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: if p.exists(): try: assert_safe_path(base, p) - logger.info(f"Found README file at {p}") + logger.debug(f"Found README file at {p}") return safe_open_text(base, p, encoding="utf-8") except Exception as e: logger.warning(f"Skipping unsafe/ unreadable README at {p}: {e}") return None - logger.info("No README file found in repository root.") + logger.debug("No README file found in repository root.") return None def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[str, Any]: @@ -277,12 +277,12 @@ def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[ - JavaScript/TypeScript AST analysis (planned) - Additional language support (future) """ - logger.info("Extracting code files from file tree...") + logger.debug("Extracting code files from file tree...") code_files = self.call_graph_analyzer.extract_code_files(file_tree) - logger.info(f"Found {len(code_files)} total code files. Filtering for supported languages.") + logger.debug(f"Found {len(code_files)} total code files. Filtering for supported languages.") supported_files = self._filter_supported_languages(code_files) - logger.info(f"Analyzing {len(supported_files)} supported files.") + logger.debug(f"Analyzing {len(supported_files)} supported files.") result = self.call_graph_analyzer.analyze_code_files(supported_files, repo_dir) @@ -321,7 +321,7 @@ def _get_supported_languages(self) -> List[str]: def _cleanup_repository(self, temp_dir: str): """Clean up cloned repository.""" - logger.info(f"Attempting to clean up {temp_dir}") + logger.debug(f"Attempting to clean up {temp_dir}") cleanup_repository(temp_dir) if temp_dir in self._temp_directories: self._temp_directories.remove(temp_dir) diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index 98cc9e31..83dbe959 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -26,7 +26,7 @@ def __init__(self, repo_path: str): self.analysis_service = AnalysisService() def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node]: - logger.info(f"Parsing repository at {self.repo_path}") + logger.debug(f"Parsing repository at {self.repo_path}") structure_result = self.analysis_service._analyze_structure( self.repo_path, @@ -41,7 +41,7 @@ def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node self._build_components_from_analysis(call_graph_result) - logger.info(f"Found {len(self.components)} components across {len(self.modules)} modules") + logger.debug(f"Found {len(self.components)} components across {len(self.modules)} modules") return self.components def _build_components_from_analysis(self, call_graph_result: Dict): @@ -141,5 +141,5 @@ def save_dependency_graph(self, output_path: str): with open(output_path, 'w', encoding='utf-8') as f: json.dump(result, f, indent=2, ensure_ascii=False) - logger.info(f"Saved {len(self.components)} components to {output_path}") + logger.debug(f"Saved {len(self.components)} components to {output_path}") return result diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index 629e1cdf..df669896 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -41,7 +41,7 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: filtered_folders = None # if os.path.exists(filtered_folders_path): - # logger.info(f"Loading filtered folders from {filtered_folders_path}") + # logger.debug(f"Loading filtered folders from {filtered_folders_path}") # filtered_folders = file_manager.load_json(filtered_folders_path) # else: # # Parse repository diff --git a/codewiki/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py index 57196313..d088fba2 100644 --- a/codewiki/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -321,7 +321,7 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes = concise_node(leaf_nodes) if len(concise_leaf_nodes) >= 400: - logger.info(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") + logger.debug(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") # Remove nodes that are dependencies of other nodes for node, deps in acyclic_graph.items(): for dep in deps: diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index 13eb91d0..aceb1a11 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -70,7 +70,7 @@ def create_documentation_metadata(self, working_dir: str, components: Dict[str, metadata_path = os.path.join(working_dir, "metadata.json") file_manager.save_json(metadata, metadata_path) - logger.info(f"Documentation metadata saved to: {metadata_path}") + logger.debug(f"Documentation metadata saved to: {metadata_path}") def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]: """Get the processing order using topological sort (leaf modules first).""" @@ -135,7 +135,7 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n # Get processing order (leaf modules first) processing_order = self.get_processing_order(first_module_tree) - logger.info(f"Processing {len(processing_order)} modules in dependency order:\n{processing_order}") + logger.debug(f"Processing {len(processing_order)} modules in dependency order:\n{processing_order}") # Process modules in dependency order final_module_tree = module_tree @@ -175,12 +175,12 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n continue # Generate repo overview - logger.info(f"Generating repo overview") + logger.debug(f"Generating repo overview") final_module_tree = await self.generate_parent_module_docs( [], working_dir ) else: - logger.info(f"Processing whole repo because repo can fit in the context window") + logger.debug(f"Processing whole repo because repo can fit in the context window") repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) final_module_tree = await self.agent_orchestrator.process_module( repo_name, components, leaf_nodes, [], working_dir @@ -197,12 +197,18 @@ async def generate_parent_module_docs(self, module_path: List[str], """Generate documentation for a parent module based on its children's documentation.""" module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path)) - logger.info(f"Generating parent documentation for: {module_name}") + logger.debug(f"Generating parent documentation for: {module_name}") # Load module tree module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) - + + # check if parent docs already exists + parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") + if os.path.exists(parent_docs_path): + logger.info(f"Parent docs already exists at {parent_docs_path}") + return module_tree + # Create repo structure with 1-depth children docs and target indicator repo_structure = self.build_overview_structure(module_tree, module_path, working_dir) @@ -220,10 +226,9 @@ async def generate_parent_module_docs(self, module_path: List[str], # Parse and save parent documentation parent_content = parent_docs.split("")[1].split("")[0].strip() # parent_content = prompt - parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") file_manager.save_text(parent_content, parent_docs_path) - logger.info(f"Successfully generated parent documentation for: {module_name}") + logger.debug(f"Successfully generated parent documentation for: {module_name}") return module_tree except Exception as e: @@ -236,8 +241,8 @@ async def run(self) -> None: # Build dependency graph components, leaf_nodes = self.graph_builder.build_dependency_graph() - logger.info(f"Found {len(leaf_nodes)} leaf nodes") - # logger.info(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") + logger.debug(f"Found {len(leaf_nodes)} leaf nodes") + # logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") # exit() # Cluster modules @@ -248,16 +253,16 @@ async def run(self) -> None: # Check if module tree exists if os.path.exists(first_module_tree_path): - logger.info(f"Module tree found at {first_module_tree_path}") + logger.debug(f"Module tree found at {first_module_tree_path}") module_tree = file_manager.load_json(first_module_tree_path) else: - logger.info(f"Module tree not found at {module_tree_path}, clustering modules") + logger.debug(f"Module tree not found at {module_tree_path}, clustering modules") module_tree = cluster_modules(leaf_nodes, components, self.config) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) - logger.info(f"Grouped components into {len(module_tree)} modules") + logger.debug(f"Grouped components into {len(module_tree)} modules") # Generate module documentation using dynamic programming approach # This processes leaf modules first, then parent modules @@ -266,9 +271,9 @@ async def run(self) -> None: # Create documentation metadata self.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - logger.info(f"Documentation generation completed successfully using dynamic programming!") - logger.info(f"Processing order: leaf modules → parent modules → repository overview") - logger.info(f"Documentation saved to: {working_dir}") + logger.debug(f"Documentation generation completed successfully using dynamic programming!") + logger.debug(f"Processing order: leaf modules → parent modules → repository overview") + logger.debug(f"Documentation saved to: {working_dir}") except Exception as e: logger.error(f"Documentation generation failed: {str(e)}") diff --git a/codewiki/src/be/main.py b/codewiki/src/be/main.py index c84e4e8f..0f24e170 100644 --- a/codewiki/src/be/main.py +++ b/codewiki/src/be/main.py @@ -50,7 +50,7 @@ async def main() -> None: await doc_generator.run() except KeyboardInterrupt: - logger.info("Documentation generation interrupted by user") + logger.debug("Documentation generation interrupted by user") except Exception as e: logger.error(f"Unexpected error: {str(e)}") raise diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py index 4723c40e..630e50d1 100644 --- a/codewiki/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -33,7 +33,7 @@ def count_tokens(text: str) -> int: Count the number of tokens in a text. """ length = len(enc.encode(text)) - logger.info(f"Number of tokens: {length}") + logger.debug(f"Number of tokens: {length}") return length @@ -76,7 +76,7 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st errors.append(error_msg) if errors: - logger.info(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") + logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") if errors: return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) @@ -140,7 +140,7 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s try: from mermaid_parser.parser import parse_mermaid_py - logger.info("Using mermaid-parser-py to validate mermaid diagrams") + logger.debug("Using mermaid-parser-py to validate mermaid diagrams") try: json_output = await parse_mermaid_py(diagram_content) @@ -160,7 +160,7 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s raise Exception(error_str) except Exception as e: - logger.info("Using mermaid-py to validate mermaid diagrams") + logger.debug("Using mermaid-py to validate mermaid diagrams") try: import mermaid as md # Create Mermaid object and check response From 05b2cb6a8fad1035a434236ad048b952e3925ca3 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:23:50 +0700 Subject: [PATCH 03/17] adjust logs --- codewiki/src/be/documentation_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index aceb1a11..d5144d1d 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -203,6 +203,12 @@ async def generate_parent_module_docs(self, module_path: List[str], module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) + # check if overview docs already exists + overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) + if os.path.exists(overview_docs_path): + logger.info(f"Overview docs already exists at {overview_docs_path}") + return module_tree + # check if parent docs already exists parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") if os.path.exists(parent_docs_path): From c04de0719ec96fe8a25428b89f931dd9a273bcaa Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:26:40 +0700 Subject: [PATCH 04/17] adjust logs --- codewiki/src/be/agent_orchestrator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codewiki/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py index 78ea3ac2..3f6d2e32 100644 --- a/codewiki/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -114,6 +114,7 @@ async def process_module(self, module_name: str, components: Dict[str, Node], # check if overview docs already exists overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) + logger.info(f"Overview docs path: {overview_docs_path}") if os.path.exists(overview_docs_path): logger.info(f"Overview docs already exists at {overview_docs_path}") return module_tree From e3f7286c60650a4b413c52d4cad484df30d74ba3 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:29:01 +0700 Subject: [PATCH 05/17] adjust logs --- codewiki/src/be/agent_orchestrator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/codewiki/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py index 3f6d2e32..78ea3ac2 100644 --- a/codewiki/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -114,7 +114,6 @@ async def process_module(self, module_name: str, components: Dict[str, Node], # check if overview docs already exists overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) - logger.info(f"Overview docs path: {overview_docs_path}") if os.path.exists(overview_docs_path): logger.info(f"Overview docs already exists at {overview_docs_path}") return module_tree From 59a2103a3288241df2338107524b42c8f34b30e3 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:32:05 +0700 Subject: [PATCH 06/17] adjust logs --- codewiki/src/be/documentation_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index d5144d1d..20113a77 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -188,7 +188,8 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n # rename repo_name.md to overview.md repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") - os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) + if os.path.exists(repo_overview_path): + os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) return working_dir From 14369ad88cf46b760563e1a50fe6790ba616951a Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:36:43 +0700 Subject: [PATCH 07/17] adjust logs --- README.md | 5 ++++- codewiki/cli/html_generator.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0846545..352b099e 100644 --- a/README.md +++ b/README.md @@ -205,4 +205,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file [⬆ Back to Top](#codewiki-automated-repository-level-documentation-generation) - \ No newline at end of file + + +/Users/anhnh/Documents/vscode/titan-sight/.venv/lib/python3.13/site-packages/codewiki/templates/github_pages/index.html +/Users/anhnh/Documents/vscode/titan-sight/.venv/lib/python3.13/site-packages/codewiki/cli/templates/github_pages/index.html \ No newline at end of file diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py index 159cc6bd..ec928c10 100644 --- a/codewiki/cli/html_generator.py +++ b/codewiki/cli/html_generator.py @@ -27,7 +27,7 @@ def __init__(self, template_dir: Optional[Path] = None): """ if template_dir is None: # Use package templates - template_dir = Path(__file__).parent / "templates" / "github_pages" + template_dir = Path(__file__).parent.parent / "templates" / "github_pages" self.template_dir = Path(template_dir) From b66bae2c5d0363747a5f2deb737eedc717aa003b Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:44:49 +0700 Subject: [PATCH 08/17] adjust docs UI --- README.md | 3 - codewiki/cli/adapters/doc_generator.py | 18 ++--- codewiki/cli/html_generator.py | 80 ++++++++++++++++++++- codewiki/templates/github_pages/app.js | 81 ++++++++++++++++++++-- codewiki/templates/github_pages/index.html | 5 ++ codewiki/templates/github_pages/styles.css | 64 ++++++++++++++++- 6 files changed, 229 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 352b099e..2cb662f4 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,3 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file [⬆ Back to Top](#codewiki-automated-repository-level-documentation-generation) - -/Users/anhnh/Documents/vscode/titan-sight/.venv/lib/python3.13/site-packages/codewiki/templates/github_pages/index.html -/Users/anhnh/Documents/vscode/titan-sight/.venv/lib/python3.13/site-packages/codewiki/cli/templates/github_pages/index.html \ No newline at end of file diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 1ed08ae1..4a933f05 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -205,26 +205,22 @@ def _run_html_generation(self): from codewiki.cli.html_generator import HTMLGenerator - # Create module tree placeholder - module_tree = { - "Overview": { - "description": "Repository overview", - "components": [], - "children": {} - } - } - # Generate HTML html_generator = HTMLGenerator() + + if self.verbose: + self.progress_tracker.update_stage(0.3, "Loading module tree and metadata...") + repo_info = html_generator.detect_repository_info(self.repo_path) + # Generate HTML with auto-loading of module_tree and metadata from docs_dir output_path = self.output_dir / "index.html" html_generator.generate( output_path=output_path, title=repo_info['name'], - module_tree=module_tree, repository_url=repo_info['url'], - github_pages_url=repo_info['github_pages_url'] + github_pages_url=repo_info['github_pages_url'], + docs_dir=self.output_dir # Auto-load module_tree and metadata from here ) self.job.files_generated.append("index.html") diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py index ec928c10..fcfae669 100644 --- a/codewiki/cli/html_generator.py +++ b/codewiki/cli/html_generator.py @@ -36,6 +36,54 @@ def __init__(self, template_dir: Optional[Path] = None): self.styles = self._load_template("styles.css") self.app_script = self._load_template("app.js") + def load_module_tree(self, docs_dir: Path) -> Dict[str, Any]: + """ + Load module tree from documentation directory. + + Args: + docs_dir: Documentation directory path + + Returns: + Module tree structure + """ + module_tree_path = docs_dir / "module_tree.json" + if not module_tree_path.exists(): + # Fallback to a simple structure + return { + "Overview": { + "description": "Repository overview", + "components": [], + "children": {} + } + } + + try: + content = safe_read(module_tree_path) + return json.loads(content) + except Exception as e: + raise FileSystemError(f"Failed to load module tree: {e}") + + def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]: + """ + Load metadata from documentation directory. + + Args: + docs_dir: Documentation directory path + + Returns: + Metadata dictionary or None if not found + """ + metadata_path = docs_dir / "metadata.json" + if not metadata_path.exists(): + return None + + try: + content = safe_read(metadata_path) + return json.loads(content) + except Exception: + # Non-critical, return None + return None + def _load_template(self, filename: str) -> str: """ Load a template file. @@ -59,10 +107,12 @@ def generate( self, output_path: Path, title: str, - module_tree: Dict[str, Any], + module_tree: Optional[Dict[str, Any]] = None, repository_url: Optional[str] = None, github_pages_url: Optional[str] = None, - config: Optional[Dict[str, Any]] = None + config: Optional[Dict[str, Any]] = None, + docs_dir: Optional[Path] = None, + metadata: Optional[Dict[str, Any]] = None ): """ Generate HTML documentation viewer. @@ -70,11 +120,31 @@ def generate( Args: output_path: Output file path (index.html) title: Documentation title - module_tree: Module tree structure + module_tree: Module tree structure (auto-loaded from docs_dir if not provided) repository_url: GitHub repository URL github_pages_url: Expected GitHub Pages URL config: Additional configuration + docs_dir: Documentation directory (for auto-loading module_tree and metadata) + metadata: Metadata dictionary (auto-loaded from docs_dir if not provided) """ + # Auto-load module tree if docs_dir provided + if docs_dir and module_tree is None: + module_tree = self.load_module_tree(docs_dir) + + # Auto-load metadata if docs_dir provided + if docs_dir and metadata is None: + metadata = self.load_metadata(docs_dir) + + # Ensure module_tree exists + if module_tree is None: + module_tree = { + "Overview": { + "description": "Repository overview", + "components": [], + "children": {} + } + } + # Build configuration full_config = { "title": title, @@ -104,6 +174,10 @@ def generate( } } + # Add metadata if available + if metadata: + full_config["metadata"] = metadata + if config: full_config.update(config) diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js index f75c1eb3..43ead9d0 100644 --- a/codewiki/templates/github_pages/app.js +++ b/codewiki/templates/github_pages/app.js @@ -85,18 +85,21 @@ } } - // Navigation tree renderer + // Navigation tree renderer - improved to handle nested structures function renderNavTree(tree, container, depth = 0) { if (!tree || typeof tree !== 'object') return; for (const [moduleName, moduleData] of Object.entries(tree)) { + // Skip if this is just metadata + if (moduleName === 'description' || moduleName === 'components') continue; + const item = document.createElement('div'); item.className = 'nav-item'; item.style.paddingLeft = `${depth * 1}rem`; const link = document.createElement('a'); link.href = `#/${moduleName}.md`; - link.textContent = moduleName; + link.textContent = formatModuleName(moduleName); link.className = 'nav-link'; // Highlight active link @@ -108,13 +111,75 @@ item.appendChild(link); container.appendChild(item); - // Render children recursively - if (moduleData.children && Object.keys(moduleData.children).length > 0) { + // Render children recursively if they exist + if (moduleData && moduleData.children && Object.keys(moduleData.children).length > 0) { renderNavTree(moduleData.children, container, depth + 1); } } } + // Format module name for display + function formatModuleName(name) { + // Replace underscores and hyphens with spaces, capitalize words + return name + .replace(/[_-]/g, ' ') + .split(' ') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + } + + // Render metadata info box + function renderMetadata(metadata) { + if (!metadata || !metadata.generation_info) return null; + + const metadataBox = document.createElement('div'); + metadataBox.style.cssText = ` + margin: 20px 0; + padding: 15px; + background: #f8fafc; + border-radius: 8px; + border: 1px solid #e2e8f0; + font-size: 11px; + line-height: 1.6; + `; + + const header = document.createElement('h4'); + header.textContent = 'GENERATION INFO'; + header.style.cssText = ` + margin: 0 0 10px 0; + font-size: 11px; + color: #64748b; + text-transform: uppercase; + letter-spacing: 0.05em; + font-weight: 600; + `; + metadataBox.appendChild(header); + + const info = metadata.generation_info; + const contentDiv = document.createElement('div'); + contentDiv.style.color = '#475569'; + + let html = ''; + if (info.main_model) { + html += `
Model: ${info.main_model}
`; + } + if (info.timestamp) { + const date = new Date(info.timestamp); + html += `
Generated: ${date.toLocaleString()}
`; + } + if (info.commit_id) { + html += `
Commit: ${info.commit_id.substring(0, 8)}
`; + } + if (metadata.statistics && metadata.statistics.total_components) { + html += `
Components: ${metadata.statistics.total_components}
`; + } + + contentDiv.innerHTML = html; + metadataBox.appendChild(contentDiv); + + return metadataBox; + } + // Mobile menu toggle function initMobileMenu() { const sidebar = document.getElementById('sidebar'); @@ -149,6 +214,14 @@ overviewItem.appendChild(overviewLink); navTree.appendChild(overviewItem); + // Add metadata info box if available + if (CODEWIKI_CONFIG && CODEWIKI_CONFIG.metadata) { + const metadataBox = renderMetadata(CODEWIKI_CONFIG.metadata); + if (metadataBox) { + navTree.appendChild(metadataBox); + } + } + // Add divider const divider = document.createElement('div'); divider.style.borderTop = '1px solid #e1e4e8'; diff --git a/codewiki/templates/github_pages/index.html b/codewiki/templates/github_pages/index.html index f1226dc7..76bce006 100644 --- a/codewiki/templates/github_pages/index.html +++ b/codewiki/templates/github_pages/index.html @@ -43,6 +43,11 @@

{{TITLE}}

+ + + diff --git a/codewiki/templates/github_pages/styles.css b/codewiki/templates/github_pages/styles.css index 615f630d..ac7b5af4 100644 --- a/codewiki/templates/github_pages/styles.css +++ b/codewiki/templates/github_pages/styles.css @@ -17,7 +17,11 @@ body { #app { display: flex; flex-direction: column; - height: 100vh; + min-height: 100vh; +} + +#footer { + margin-top: auto; } /* Header */ @@ -89,6 +93,7 @@ body { text-decoration: none; border-radius: 4px; transition: background 0.2s; + font-size: 0.875rem; } .nav-link:hover { @@ -100,6 +105,25 @@ body { color: #ffffff; } +/* Support for deeply nested navigation items */ +.nav-item[style*="padding-left: 2rem"] .nav-link { + font-size: 0.8125rem; + color: #586069; +} + +.nav-item[style*="padding-left: 3rem"] .nav-link { + font-size: 0.75rem; + color: #6a737d; +} + +.metadata-box { + margin: 1rem 0; + padding: 1rem; + background: #f8fafc; + border-radius: 8px; + border: 1px solid #e2e8f0; +} + /* Content */ #content { flex: 1; @@ -214,6 +238,44 @@ body { margin: 0.5rem 0; } +#markdown-content blockquote { + border-left: 4px solid #0366d6; + padding: 0.5rem 1rem; + margin: 1rem 0; + background: #f6f8fa; + color: #586069; +} + +#markdown-content img { + max-width: 100%; + height: auto; + border-radius: 6px; + margin: 1rem 0; +} + +#markdown-content hr { + border: none; + border-top: 1px solid #e1e4e8; + margin: 2rem 0; +} + +/* Mermaid diagrams */ +.mermaid { + background: #ffffff; + border: 1px solid #e1e4e8; + border-radius: 6px; + padding: 1rem; + margin: 1rem 0; + text-align: center; +} + +/* Loading state */ +.loading { + text-align: center; + padding: 2rem; + color: #586069; +} + /* Responsive Design */ @media (max-width: 768px) { #sidebar { From 1dd210c4eb9cf2d991bc7644ebc888a3c91c145e Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:49:18 +0700 Subject: [PATCH 09/17] adjust docs UI --- codewiki/src/be/documentation_generator.py | 3 ++ codewiki/templates/github_pages/app.js | 58 +++++++++++++++++----- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index 20113a77..86018273 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -186,6 +186,9 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n repo_name, components, leaf_nodes, [], working_dir ) + # save final_module_tree to module_tree.json + file_manager.save_json(final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)) + # rename repo_name.md to overview.md repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") if os.path.exists(repo_overview_path): diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js index 43ead9d0..5241a4e2 100644 --- a/codewiki/templates/github_pages/app.js +++ b/codewiki/templates/github_pages/app.js @@ -28,24 +28,54 @@ handleRoute() { const hash = window.location.hash.slice(1) || '/'; - const path = hash === '/' ? 'overview.md' : hash; + // Remove leading slash if present + const cleanHash = hash.startsWith('/') ? hash.slice(1) : hash; + const path = (hash === '/' || cleanHash === '') ? 'overview.md' : cleanHash; this.loadMarkdown(path); this.updateBreadcrumbs(path); } async loadMarkdown(filename) { + const content = document.getElementById('markdown-content'); + + // Show loading state + content.innerHTML = '
Loading...
'; + try { const response = await fetch(filename); - if (!response.ok) throw new Error(`Failed to load ${filename}`); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } const markdown = await response.text(); this.renderMarkdown(markdown); + this.updateActiveLink(filename); } catch (error) { console.error('Error loading markdown:', error); - const content = document.getElementById('markdown-content'); - content.innerHTML = `

Error

Could not load ${filename}

`; + content.innerHTML = ` +

📄 Document Not Found

+

Could not load ${filename}

+

+ ${error.message} +

+

+ ← Back to Overview +

+ `; } } + updateActiveLink(filename) { + // Update active state on navigation links + document.querySelectorAll('.nav-link').forEach(link => { + link.classList.remove('active'); + const linkPath = link.getAttribute('href').replace('#/', '').replace('#', ''); + const currentPath = filename; + if (linkPath === currentPath || (linkPath === '' && filename === 'overview.md')) { + link.classList.add('active'); + } + }); + } + renderMarkdown(markdown) { const html = marked.parse(markdown); const content = document.getElementById('markdown-content'); @@ -70,15 +100,23 @@ updateBreadcrumbs(path) { const breadcrumbs = document.getElementById('breadcrumbs'); + + // Handle overview case + if (path === 'overview.md') { + breadcrumbs.innerHTML = 'Home'; + return; + } + const parts = path.split('/').filter(p => p); let html = 'Home'; let currentPath = ''; for (const part of parts) { - currentPath += '/' + part; + currentPath += (currentPath ? '/' : '') + part; html += ' › '; - html += `${part.replace('.md', '')}`; + const displayName = formatModuleName(part.replace('.md', '')); + html += `${displayName}`; } breadcrumbs.innerHTML = html; @@ -102,11 +140,7 @@ link.textContent = formatModuleName(moduleName); link.className = 'nav-link'; - // Highlight active link - link.addEventListener('click', function() { - document.querySelectorAll('.nav-link').forEach(l => l.classList.remove('active')); - this.classList.add('active'); - }); + // Active link highlighting is handled by updateActiveLink in the router item.appendChild(link); container.appendChild(item); @@ -210,7 +244,7 @@ const overviewLink = document.createElement('a'); overviewLink.href = '#/'; overviewLink.textContent = 'Overview'; - overviewLink.className = 'nav-link active'; + overviewLink.className = 'nav-link'; // Active state will be set by router overviewItem.appendChild(overviewLink); navTree.appendChild(overviewItem); From 33f1537a443ae0c71f0804e016cdbd401d588f29 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 22:58:14 +0700 Subject: [PATCH 10/17] adjust docs UI --- codewiki/cli/commands/generate.py | 4 ++++ codewiki/src/be/utils.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index 7a9234ad..a4a3e09f 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -3,6 +3,7 @@ """ import sys +import logging from pathlib import Path from typing import Optional import click @@ -90,6 +91,9 @@ def generate_command( logger = create_logger(verbose=verbose) start_time = time.time() + # Suppress httpx INFO logs + logging.getLogger("httpx").setLevel(logging.WARNING) + try: # Pre-generation checks logger.step("Validating configuration...", 1, 4) diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py index 630e50d1..953c8047 100644 --- a/codewiki/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -135,6 +135,9 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s Returns: Error message if invalid, empty string if valid """ + import sys + import os + from io import StringIO core_error = "" @@ -143,7 +146,16 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s logger.debug("Using mermaid-parser-py to validate mermaid diagrams") try: - json_output = await parse_mermaid_py(diagram_content) + # Redirect stderr to suppress mermaid parser JavaScript errors + old_stderr = sys.stderr + sys.stderr = open(os.devnull, 'w') + + try: + json_output = await parse_mermaid_py(diagram_content) + finally: + # Restore stderr + sys.stderr.close() + sys.stderr = old_stderr except Exception as e: error_str = str(e) From cf06340500585b6d2b8e536197e61da9f4827a83 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 23:13:48 +0700 Subject: [PATCH 11/17] adjust docs UI --- codewiki/templates/github_pages/app.js | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js index 5241a4e2..e092bbff 100644 --- a/codewiki/templates/github_pages/app.js +++ b/codewiki/templates/github_pages/app.js @@ -94,10 +94,35 @@ mermaid.run(); } + // Intercept clicks on markdown links and convert to hash-based navigation + this.interceptMarkdownLinks(content); + // Scroll to top document.getElementById('content').scrollTop = 0; } + interceptMarkdownLinks(container) { + // Find all links within the markdown content + const links = container.querySelectorAll('a[href]'); + + links.forEach(link => { + const href = link.getAttribute('href'); + + // Only intercept links to .md files (internal documentation links) + if (href && href.endsWith('.md') && !href.startsWith('http://') && !href.startsWith('https://')) { + link.addEventListener('click', (e) => { + e.preventDefault(); + + // Remove leading './' or '/' if present + let cleanPath = href.replace(/^\.\//, '').replace(/^\//, ''); + + // Update hash to trigger navigation + window.location.hash = `/${cleanPath}`; + }); + } + }); + } + updateBreadcrumbs(path) { const breadcrumbs = document.getElementById('breadcrumbs'); @@ -140,8 +165,6 @@ link.textContent = formatModuleName(moduleName); link.className = 'nav-link'; - // Active link highlighting is handled by updateActiveLink in the router - item.appendChild(link); container.appendChild(item); @@ -244,7 +267,7 @@ const overviewLink = document.createElement('a'); overviewLink.href = '#/'; overviewLink.textContent = 'Overview'; - overviewLink.className = 'nav-link'; // Active state will be set by router + overviewLink.className = 'nav-link'; overviewItem.appendChild(overviewLink); navTree.appendChild(overviewItem); From c18a36ca9317e34f34475da0e3ab6bf1e3b5b154 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sat, 18 Oct 2025 23:21:43 +0700 Subject: [PATCH 12/17] adjust docs UI --- codewiki/templates/github_pages/app.js | 311 +++++++++++-------- codewiki/templates/github_pages/index.html | 50 +-- codewiki/templates/github_pages/styles.css | 338 +++++++++------------ 3 files changed, 346 insertions(+), 353 deletions(-) diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js index e092bbff..8a96045c 100644 --- a/codewiki/templates/github_pages/app.js +++ b/codewiki/templates/github_pages/app.js @@ -3,20 +3,127 @@ (function() { 'use strict'; - // Initialize marked - marked.setOptions({ - gfm: true, - breaks: true, - highlight: function(code, lang) { - if (lang && hljs.getLanguage(lang)) { - return hljs.highlight(code, { language: lang }).value; + // Simple markdown parser (GitHub-flavored) + function parseMarkdown(markdown) { + let html = markdown; + + // Escape HTML + html = html.replace(/&/g, '&').replace(//g, '>'); + + // Headers + html = html.replace(/^##### (.*$)/gim, '
$1
'); + html = html.replace(/^#### (.*$)/gim, '

$1

'); + html = html.replace(/^### (.*$)/gim, '

$1

'); + html = html.replace(/^## (.*$)/gim, '

$1

'); + html = html.replace(/^# (.*$)/gim, '

$1

'); + + // Bold + html = html.replace(/\*\*(.+?)\*\*/g, '$1'); + html = html.replace(/__(.+?)__/g, '$1'); + + // Italic + html = html.replace(/\*(.+?)\*/g, '$1'); + html = html.replace(/_(.+?)_/g, '$1'); + + // Code blocks with language support (including mermaid) + html = html.replace(/```(\w+)?\n([\s\S]*?)```/g, function(match, lang, code) { + code = code.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'); + if (lang === 'mermaid') { + return '
' + code + '
'; } - return hljs.highlightAuto(code).value; - } - }); + return '
' + code.replace(//g, '>') + '
'; + }); + + // Inline code + html = html.replace(/`([^`]+)`/g, '$1'); + + // Links + html = html.replace(/\[([^\]]+)\]\(([^\)]+)\)/g, '$1'); + + // Images + html = html.replace(/!\[([^\]]*)\]\(([^\)]+)\)/g, '$1'); + + // Blockquotes + html = html.replace(/^\> (.+)$/gim, '
$1
'); + + // Horizontal rules + html = html.replace(/^---$/gim, '
'); + html = html.replace(/^\*\*\*$/gim, '
'); + + // Unordered lists + html = html.replace(/^\* (.+)$/gim, '
  • $1
    • '); + html = html.replace(/^- (.+)$/gim, '
  • $1
    • '); + html = html.replace(/(
  • .*<\/li>)/s, '
      $1
    '); + + // Ordered lists + html = html.replace(/^\d+\. (.+)$/gim, '
  • $1
    • '); + + // Tables (basic support) + html = html.replace(/\|(.+)\|/g, function(match) { + const cells = match.split('|').filter(cell => cell.trim()); + return '' + cells.map(cell => '' + cell.trim() + '').join('') + ''; + }); + html = html.replace(/(.*<\/tr>)/s, '$1
    '); + + // Paragraphs + html = html.replace(/\n\n/g, '

    '); + html = '

    ' + html + '

    '; + + // Clean up + html = html.replace(/

    <\/p>/g, ''); + html = html.replace(/

    ()/g, '$1'); + html = html.replace(/(<\/h[1-6]>)<\/p>/g, '$1'); + html = html.replace(/

    ()/g, '$1'); + html = html.replace(/(<\/table>)<\/p>/g, '$1'); + html = html.replace(/

    (

      )/g, '$1'); + html = html.replace(/(<\/ul>)<\/p>/g, '$1'); + html = html.replace(/

      (

      )/g, '$1');
+        html = html.replace(/(<\/pre>)<\/p>/g, '$1');
+        html = html.replace(/
      (
      )/g, '$1');
+        html = html.replace(/(<\/div>)<\/p>/g, '$1');
+        html = html.replace(/
      (
      )<\/p>/g, '$1');
+        html = html.replace(/
      (
      )/g, '$1');
+        html = html.replace(/(<\/blockquote>)<\/p>/g, '$1');
+        
+        return html;
+    }
     
     // Initialize mermaid
-    mermaid.initialize({ startOnLoad: false, theme: 'default' });
+    mermaid.initialize({
+        startOnLoad: false,
+        theme: 'default',
+        themeVariables: {
+            primaryColor: '#2563eb',
+            primaryTextColor: '#334155',
+            primaryBorderColor: '#e2e8f0',
+            lineColor: '#64748b',
+            sectionBkgColor: '#f8fafc',
+            altSectionBkgColor: '#f1f5f9',
+            gridColor: '#e2e8f0',
+            secondaryColor: '#f1f5f9',
+            tertiaryColor: '#f8fafc'
+        },
+        flowchart: {
+            htmlLabels: true,
+            curve: 'basis'
+        },
+        sequence: {
+            diagramMarginX: 50,
+            diagramMarginY: 10,
+            actorMargin: 50,
+            width: 150,
+            height: 65,
+            boxMargin: 10,
+            boxTextMargin: 5,
+            noteMargin: 10,
+            messageMargin: 35,
+            mirrorActors: true,
+            bottomMarginAdj: 1,
+            useMaxWidth: true,
+            rightAngles: false,
+            showSequenceNumbers: false
+        }
+    });
     
     // Router
     class Router {
@@ -28,11 +135,9 @@
         
         handleRoute() {
             const hash = window.location.hash.slice(1) || '/';
-            // Remove leading slash if present
             const cleanHash = hash.startsWith('/') ? hash.slice(1) : hash;
             const path = (hash === '/' || cleanHash === '') ? 'overview.md' : cleanHash;
             this.loadMarkdown(path);
-            this.updateBreadcrumbs(path);
         }
         
         async loadMarkdown(filename) {
@@ -54,18 +159,17 @@
                 content.innerHTML = `
                     
      📄 Document Not Found
      
                     
      Could not load ${filename}
      
-                    
      
+                    
      
                         ${error.message}
                     
      
                     
      
-                        ← Back to Overview
+                        ← Back to Overview
                     
      
                 `;
             }
         }
         
         updateActiveLink(filename) {
-            // Update active state on navigation links
             document.querySelectorAll('.nav-link').forEach(link => {
                 link.classList.remove('active');
                 const linkPath = link.getAttribute('href').replace('#/', '').replace('#', '');
@@ -76,108 +180,75 @@
             });
         }
         
-        renderMarkdown(markdown) {
-            const html = marked.parse(markdown);
+        async renderMarkdown(markdown) {
+            const html = parseMarkdown(markdown);
             const content = document.getElementById('markdown-content');
             content.innerHTML = html;
             
             // Process mermaid diagrams
-            const mermaidBlocks = content.querySelectorAll('code.language-mermaid');
-            mermaidBlocks.forEach((block, index) => {
-                const div = document.createElement('div');
-                div.className = 'mermaid';
-                div.textContent = block.textContent;
-                block.parentElement.replaceWith(div);
-            });
-            
-            if (mermaidBlocks.length > 0) {
-                mermaid.run();
-            }
-            
-            // Intercept clicks on markdown links and convert to hash-based navigation
-            this.interceptMarkdownLinks(content);
-            
-            // Scroll to top
-            document.getElementById('content').scrollTop = 0;
-        }
-        
-        interceptMarkdownLinks(container) {
-            // Find all links within the markdown content
-            const links = container.querySelectorAll('a[href]');
-            
-            links.forEach(link => {
-                const href = link.getAttribute('href');
-                
-                // Only intercept links to .md files (internal documentation links)
-                if (href && href.endsWith('.md') && !href.startsWith('http://') && !href.startsWith('https://')) {
-                    link.addEventListener('click', (e) => {
-                        e.preventDefault();
-                        
-                        // Remove leading './' or '/' if present
-                        let cleanPath = href.replace(/^\.\//, '').replace(/^\//, '');
-                        
-                        // Update hash to trigger navigation
-                        window.location.hash = `/${cleanPath}`;
+            const mermaidDivs = content.querySelectorAll('.mermaid');
+            if (mermaidDivs.length > 0) {
+                try {
+                    await mermaid.run({
+                        nodes: mermaidDivs
                     });
+                } catch (e) {
+                    console.error('Mermaid rendering error:', e);
                 }
-            });
-        }
-        
-        updateBreadcrumbs(path) {
-            const breadcrumbs = document.getElementById('breadcrumbs');
-            
-            // Handle overview case
-            if (path === 'overview.md') {
-                breadcrumbs.innerHTML = 'Home';
-                return;
             }
             
-            const parts = path.split('/').filter(p => p);
-            
-            let html = 'Home';
-            let currentPath = '';
-            
-            for (const part of parts) {
-                currentPath += (currentPath ? '/' : '') + part;
-                html += ' › ';
-                const displayName = formatModuleName(part.replace('.md', ''));
-                html += `${displayName}`;
-            }
-            
-            breadcrumbs.innerHTML = html;
+            // Scroll to top
+            window.scrollTo(0, 0);
         }
     }
     
-    // Navigation tree renderer - improved to handle nested structures
+    // Navigation tree renderer with proper subsection support
     function renderNavTree(tree, container, depth = 0) {
         if (!tree || typeof tree !== 'object') return;
         
         for (const [moduleName, moduleData] of Object.entries(tree)) {
-            // Skip if this is just metadata
             if (moduleName === 'description' || moduleName === 'components') continue;
             
-            const item = document.createElement('div');
-            item.className = 'nav-item';
-            item.style.paddingLeft = `${depth * 1}rem`;
-            
-            const link = document.createElement('a');
-            link.href = `#/${moduleName}.md`;
-            link.textContent = formatModuleName(moduleName);
-            link.className = 'nav-link';
+            // Check if this has components (is a page) or just children (is a section)
+            const hasComponents = moduleData && moduleData.components && moduleData.components.length > 0;
+            const hasChildren = moduleData && moduleData.children && Object.keys(moduleData.children).length > 0;
             
-            item.appendChild(link);
-            container.appendChild(item);
+            if (hasComponents) {
+                // This is a page - render as link
+                const item = document.createElement('div');
+                item.className = depth > 0 ? 'nav-subsection' : 'nav-item';
+                
+                const link = document.createElement('a');
+                link.href = `#/${moduleName}.md`;
+                link.textContent = formatModuleName(moduleName);
+                link.className = 'nav-link';
+                
+                item.appendChild(link);
+                container.appendChild(item);
+            } else if (hasChildren) {
+                // This is a section header - render as text
+                const sectionHeader = document.createElement('div');
+                sectionHeader.className = 'nav-section-header';
+                sectionHeader.textContent = formatModuleName(moduleName);
+                if (depth > 0) {
+                    sectionHeader.style.fontSize = `${14 - (depth * 1)}px`;
+                    sectionHeader.style.textTransform = 'none';
+                }
+                container.appendChild(sectionHeader);
+            }
             
-            // Render children recursively if they exist
-            if (moduleData && moduleData.children && Object.keys(moduleData.children).length > 0) {
-                renderNavTree(moduleData.children, container, depth + 1);
+            // Render children recursively
+            if (hasChildren) {
+                const subsection = document.createElement('div');
+                subsection.className = 'nav-subsection';
+                container.appendChild(subsection);
+                renderNavTree(moduleData.children, subsection, depth + 1);
             }
         }
     }
     
     // Format module name for display
     function formatModuleName(name) {
-        // Replace underscores and hyphens with spaces, capitalize words
         return name
             .replace(/[_-]/g, ' ')
             .split(' ')
@@ -196,25 +267,22 @@
             background: #f8fafc;
             border-radius: 8px;
             border: 1px solid #e2e8f0;
-            font-size: 11px;
-            line-height: 1.6;
         `;
         
         const header = document.createElement('h4');
         header.textContent = 'GENERATION INFO';
         header.style.cssText = `
             margin: 0 0 10px 0;
-            font-size: 11px;
+            font-size: 12px;
             color: #64748b;
             text-transform: uppercase;
             letter-spacing: 0.05em;
-            font-weight: 600;
         `;
         metadataBox.appendChild(header);
         
         const info = metadata.generation_info;
         const contentDiv = document.createElement('div');
-        contentDiv.style.color = '#475569';
+        contentDiv.style.cssText = 'font-size: 11px; color: #475569; line-height: 1.4;';
         
         let html = '';
         if (info.main_model) {
@@ -222,7 +290,7 @@
         }
         if (info.timestamp) {
             const date = new Date(info.timestamp);
-            html += `
      Generated: ${date.toLocaleString()}
      `;
+            html += `
      Generated: ${date.toLocaleString().substring(0, 16)}
      `;
         }
         if (info.commit_id) {
             html += `
      Commit: ${info.commit_id.substring(0, 8)}
      `;
@@ -237,31 +305,15 @@
         return metadataBox;
     }
     
-    // Mobile menu toggle
-    function initMobileMenu() {
-        const sidebar = document.getElementById('sidebar');
-        const toggleBtn = document.getElementById('mobile-menu-toggle');
-        
-        toggleBtn.addEventListener('click', () => {
-            sidebar.classList.toggle('open');
-        });
-        
-        // Close sidebar when clicking outside on mobile
-        document.addEventListener('click', (e) => {
-            if (window.innerWidth < 768 && 
-                !sidebar.contains(e.target) && 
-                e.target !== toggleBtn) {
-                sidebar.classList.remove('open');
-            }
-        });
-    }
-    
     // Initialize application
     function init() {
-        // Render navigation tree
         const navTree = document.getElementById('nav-tree');
+        
         if (MODULE_TREE) {
-            // Add overview link first
+            // Add overview link
+            const navSection = document.createElement('div');
+            navSection.className = 'nav-section';
+            
             const overviewItem = document.createElement('div');
             overviewItem.className = 'nav-item';
             const overviewLink = document.createElement('a');
@@ -269,32 +321,27 @@
             overviewLink.textContent = 'Overview';
             overviewLink.className = 'nav-link';
             overviewItem.appendChild(overviewLink);
-            navTree.appendChild(overviewItem);
+            navSection.appendChild(overviewItem);
+            navTree.appendChild(navSection);
             
             // Add metadata info box if available
             if (CODEWIKI_CONFIG && CODEWIKI_CONFIG.metadata) {
                 const metadataBox = renderMetadata(CODEWIKI_CONFIG.metadata);
                 if (metadataBox) {
-                    navTree.appendChild(metadataBox);
+                    document.getElementById('metadata-info').appendChild(metadataBox);
                 }
             }
             
-            // Add divider
-            const divider = document.createElement('div');
-            divider.style.borderTop = '1px solid #e1e4e8';
-            divider.style.margin = '0.5rem 0';
-            navTree.appendChild(divider);
-            
             // Render module tree
-            renderNavTree(MODULE_TREE, navTree);
+            const moduleSection = document.createElement('div');
+            moduleSection.className = 'nav-section';
+            navTree.appendChild(moduleSection);
+            renderNavTree(MODULE_TREE, moduleSection);
         }
         
         // Initialize router
         const router = new Router();
         
-        // Initialize mobile menu
-        initMobileMenu();
-        
         // Initial load
         router.handleRoute();
     }
diff --git a/codewiki/templates/github_pages/index.html b/codewiki/templates/github_pages/index.html
index 76bce006..c711b30e 100644
--- a/codewiki/templates/github_pages/index.html
+++ b/codewiki/templates/github_pages/index.html
@@ -3,51 +3,31 @@
 
     
     
-    {{TITLE}} - Documentation
+    {{TITLE}}
     
     
     
-    
-    
-    
+    
     
     
 
 
-    
      
-        
-        
-
-        
-        
      
-            
-            
-
-            
-            
      
-                
-                
      
-                    
      Loading...
      
-                
      
-            
      
-        
      
+    
      
+        
         
-        
-        
+        
      
+            
      
+                
      Loading...
      
+            
      
+        
      
     
      
 
     
diff --git a/codewiki/templates/github_pages/styles.css b/codewiki/templates/github_pages/styles.css
index ac7b5af4..c2ef7da6 100644
--- a/codewiki/templates/github_pages/styles.css
+++ b/codewiki/templates/github_pages/styles.css
@@ -1,4 +1,11 @@
-/* Reset and Base Styles */
+:root {
+    --primary-color: #2563eb;
+    --secondary-color: #f1f5f9;
+    --text-color: #334155;
+    --border-color: #e2e8f0;
+    --hover-color: #f8fafc;
+}
+
 * {
     margin: 0;
     padding: 0;
@@ -6,263 +13,236 @@
 }
 
 body {
-    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
-                 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
-                 sans-serif;
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
     line-height: 1.6;
-    color: #24292e;
-    background: #ffffff;
+    color: var(--text-color);
+    background-color: #ffffff;
 }
 
-#app {
+.container {
     display: flex;
-    flex-direction: column;
     min-height: 100vh;
 }
 
-#footer {
-    margin-top: auto;
-}
-
-/* Header */
-#header {
-    background: #24292e;
-    color: #ffffff;
-    padding: 1rem;
-    box-shadow: 0 2px 4px rgba(0,0,0,0.1);
-    position: relative;
-}
-
-.header-content {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
+.sidebar {
+    width: 300px;
+    background-color: var(--secondary-color);
+    border-right: 1px solid var(--border-color);
+    padding: 20px;
+    overflow-y: auto;
+    position: fixed;
+    height: 100vh;
 }
 
-#doc-title {
-    font-size: 1.5rem;
-    font-weight: 600;
+.content {
+    flex: 1;
+    margin-left: 300px;
+    padding: 40px 60px;
+    max-width: calc(100% - 300px);
 }
 
-#header-nav a {
-    color: #ffffff;
+.logo {
+    font-size: 24px;
+    font-weight: bold;
+    color: var(--primary-color);
+    margin-bottom: 30px;
     text-decoration: none;
-    padding: 0.5rem 1rem;
-    border-radius: 4px;
-    transition: background 0.2s;
-}
-
-#header-nav a:hover {
-    background: rgba(255,255,255,0.1);
+    display: block;
 }
 
-#mobile-menu-toggle {
-    display: none;
-    background: none;
-    border: none;
-    color: #ffffff;
-    font-size: 1.5rem;
-    cursor: pointer;
-    padding: 0.5rem;
+.nav-section {
+    margin-bottom: 25px;
 }
 
-/* Main Container */
-#main-container {
-    display: flex;
-    flex: 1;
-    overflow: hidden;
-}
-
-/* Sidebar */
-#sidebar {
-    width: 280px;
-    background: #f6f8fa;
-    border-right: 1px solid #e1e4e8;
-    overflow-y: auto;
-    padding: 1rem;
+.nav-section h3 {
+    font-size: 14px;
+    font-weight: 600;
+    color: #64748b;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    margin-bottom: 10px;
 }
 
 .nav-item {
-    margin: 0.25rem 0;
+    display: block;
+    margin-bottom: 2px;
 }
 
 .nav-link {
     display: block;
-    padding: 0.5rem;
-    color: #0366d6;
+    padding: 8px 12px;
+    color: var(--text-color);
     text-decoration: none;
-    border-radius: 4px;
-    transition: background 0.2s;
-    font-size: 0.875rem;
+    border-radius: 6px;
+    font-size: 14px;
+    transition: all 0.2s ease;
 }
 
 .nav-link:hover {
-    background: #e1e4e8;
+    background-color: var(--hover-color);
+    color: var(--primary-color);
 }
 
 .nav-link.active {
-    background: #0366d6;
-    color: #ffffff;
+    background-color: var(--primary-color);
+    color: white;
 }
 
-/* Support for deeply nested navigation items */
-.nav-item[style*="padding-left: 2rem"] .nav-link {
-    font-size: 0.8125rem;
-    color: #586069;
+.nav-subsection {
+    margin-left: 15px;
+    margin-top: 8px;
 }
 
-.nav-item[style*="padding-left: 3rem"] .nav-link {
-    font-size: 0.75rem;
-    color: #6a737d;
+.nav-subsection .nav-link {
+    font-size: 13px;
+    color: #64748b;
 }
 
-.metadata-box {
-    margin: 1rem 0;
-    padding: 1rem;
-    background: #f8fafc;
-    border-radius: 8px;
-    border: 1px solid #e2e8f0;
+.nav-section-header {
+    font-size: 14px;
+    font-weight: 600;
+    color: #64748b;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+    margin-bottom: 10px;
+    padding: 8px 12px;
 }
 
-/* Content */
-#content {
-    flex: 1;
-    overflow-y: auto;
-    padding: 2rem;
-    max-width: 900px;
-    margin: 0 auto;
-    width: 100%;
+/* Nested subsection indentation - scalable for any depth */
+.nav-subsection .nav-subsection {
+    margin-left: 20px;
 }
 
-#breadcrumbs {
-    margin-bottom: 1rem;
-    color: #586069;
-    font-size: 0.9rem;
+.nav-subsection .nav-subsection .nav-link {
+    font-size: 12px;
 }
 
-#breadcrumbs a {
-    color: #0366d6;
-    text-decoration: none;
+/* Additional nesting levels */
+.nav-subsection .nav-subsection .nav-subsection {
+    margin-left: 15px;
 }
 
-#breadcrumbs a:hover {
-    text-decoration: underline;
+.nav-subsection .nav-subsection .nav-subsection .nav-link {
+    font-size: 11px;
 }
 
-.separator {
-    margin: 0 0.5rem;
+.markdown-content {
+    max-width: none;
 }
 
-/* Markdown Content */
-#markdown-content {
-    line-height: 1.8;
+.markdown-content h1 {
+    font-size: 2.5rem;
+    font-weight: 700;
+    color: #1e293b;
+    margin-bottom: 1rem;
+    border-bottom: 2px solid var(--border-color);
+    padding-bottom: 0.5rem;
 }
 
-#markdown-content h1,
-#markdown-content h2,
-#markdown-content h3 {
-    margin-top: 1.5rem;
+.markdown-content h2 {
+    font-size: 2rem;
+    font-weight: 600;
+    color: #334155;
+    margin-top: 2rem;
     margin-bottom: 1rem;
+}
+
+.markdown-content h3 {
+    font-size: 1.5rem;
     font-weight: 600;
+    color: #475569;
+    margin-top: 1.5rem;
+    margin-bottom: 0.75rem;
 }
 
-#markdown-content h1 {
-    font-size: 2rem;
-    border-bottom: 1px solid #e1e4e8;
-    padding-bottom: 0.5rem;
+.markdown-content p {
+    margin-bottom: 1rem;
+    color: #475569;
 }
 
-#markdown-content h2 {
-    font-size: 1.5rem;
+.markdown-content ul, .markdown-content ol {
+    margin-bottom: 1rem;
+    padding-left: 1.5rem;
 }
 
-#markdown-content h3 {
-    font-size: 1.25rem;
+.markdown-content li {
+    margin-bottom: 0.5rem;
+    color: #475569;
 }
 
-#markdown-content code {
-    background: #f6f8fa;
-    padding: 0.2rem 0.4rem;
-    border-radius: 3px;
-    font-family: 'Courier New', monospace;
-    font-size: 0.9em;
+.markdown-content code {
+    background-color: #f1f5f9;
+    padding: 0.25rem 0.5rem;
+    border-radius: 0.25rem;
+    font-family: 'Fira Code', 'Consolas', monospace;
+    font-size: 0.875rem;
 }
 
-#markdown-content pre {
-    background: #f6f8fa;
+.markdown-content pre {
+    background-color: #f8fafc;
+    border: 1px solid var(--border-color);
+    border-radius: 0.5rem;
     padding: 1rem;
-    border-radius: 6px;
     overflow-x: auto;
-    margin: 1rem 0;
+    margin-bottom: 1rem;
 }
 
-#markdown-content pre code {
-    background: none;
+.markdown-content pre code {
+    background-color: transparent;
     padding: 0;
 }
 
-#markdown-content table {
-    border-collapse: collapse;
+.markdown-content blockquote {
+    border-left: 4px solid var(--primary-color);
+    padding-left: 1rem;
+    margin-bottom: 1rem;
+    font-style: italic;
+    color: #64748b;
+}
+
+.markdown-content table {
     width: 100%;
-    margin: 1rem 0;
+    border-collapse: collapse;
+    margin-bottom: 1rem;
 }
 
-#markdown-content th,
-#markdown-content td {
-    border: 1px solid #e1e4e8;
-    padding: 0.5rem;
+.markdown-content th, .markdown-content td {
+    border: 1px solid var(--border-color);
+    padding: 0.75rem;
     text-align: left;
 }
 
-#markdown-content th {
-    background: #f6f8fa;
+.markdown-content th {
+    background-color: var(--secondary-color);
     font-weight: 600;
 }
 
-#markdown-content a {
-    color: #0366d6;
-    text-decoration: none;
-}
-
-#markdown-content a:hover {
+.markdown-content a {
+    color: var(--primary-color);
     text-decoration: underline;
 }
 
-#markdown-content ul,
-#markdown-content ol {
-    margin: 1rem 0;
-    padding-left: 2rem;
-}
-
-#markdown-content li {
-    margin: 0.5rem 0;
-}
-
-#markdown-content blockquote {
-    border-left: 4px solid #0366d6;
-    padding: 0.5rem 1rem;
-    margin: 1rem 0;
-    background: #f6f8fa;
-    color: #586069;
+.markdown-content a:hover {
+    text-decoration: none;
 }
 
-#markdown-content img {
+.markdown-content img {
     max-width: 100%;
     height: auto;
-    border-radius: 6px;
+    border-radius: 0.5rem;
     margin: 1rem 0;
 }
 
-#markdown-content hr {
+.markdown-content hr {
     border: none;
-    border-top: 1px solid #e1e4e8;
+    border-top: 1px solid var(--border-color);
     margin: 2rem 0;
 }
 
 /* Mermaid diagrams */
 .mermaid {
     background: #ffffff;
-    border: 1px solid #e1e4e8;
+    border: 1px solid var(--border-color);
     border-radius: 6px;
     padding: 1rem;
     margin: 1rem 0;
@@ -273,34 +253,20 @@ body {
 .loading {
     text-align: center;
     padding: 2rem;
-    color: #586069;
+    color: #64748b;
 }
 
-/* Responsive Design */
 @media (max-width: 768px) {
-    #sidebar {
-        position: fixed;
-        left: -280px;
-        top: 60px;
-        height: calc(100vh - 60px);
-        z-index: 1000;
-        transition: left 0.3s;
-    }
-    
-    #sidebar.open {
-        left: 0;
-    }
-    
-    #mobile-menu-toggle {
-        display: block;
-        position: absolute;
-        right: 1rem;
-        top: 50%;
-        transform: translateY(-50%);
+    .sidebar {
+        width: 100%;
+        position: relative;
+        height: auto;
     }
     
-    #content {
-        padding: 1rem;
+    .content {
+        margin-left: 0;
+        padding: 20px;
+        max-width: 100%;
     }
 }
 

From d2762445e64367bd9c889ccaa495ed55113bd9d3 Mon Sep 17 00:00:00 2001
From: anhnh2002 
Date: Sat, 18 Oct 2025 23:27:05 +0700
Subject: [PATCH 13/17] adjust docs UI

---
 codewiki/templates/github_pages/app.js     | 154 +++++----------------
 codewiki/templates/github_pages/index.html |   1 +
 2 files changed, 38 insertions(+), 117 deletions(-)

diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js
index 8a96045c..a01deb6f 100644
--- a/codewiki/templates/github_pages/app.js
+++ b/codewiki/templates/github_pages/app.js
@@ -3,89 +3,12 @@
 (function() {
     'use strict';
     
-    // Simple markdown parser (GitHub-flavored)
-    function parseMarkdown(markdown) {
-        let html = markdown;
-        
-        // Escape HTML
-        html = html.replace(/&/g, '&').replace(//g, '>');
-        
-        // Headers
-        html = html.replace(/^##### (.*$)/gim, '
      $1
      ');
-        html = html.replace(/^#### (.*$)/gim, '
      $1
      ');
-        html = html.replace(/^### (.*$)/gim, '
      $1
      ');
-        html = html.replace(/^## (.*$)/gim, '
      $1
      ');
-        html = html.replace(/^# (.*$)/gim, '
      $1
      ');
-        
-        // Bold
-        html = html.replace(/\*\*(.+?)\*\*/g, '$1');
-        html = html.replace(/__(.+?)__/g, '$1');
-        
-        // Italic
-        html = html.replace(/\*(.+?)\*/g, '$1');
-        html = html.replace(/_(.+?)_/g, '$1');
-        
-        // Code blocks with language support (including mermaid)
-        html = html.replace(/```(\w+)?\n([\s\S]*?)```/g, function(match, lang, code) {
-            code = code.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
-            if (lang === 'mermaid') {
-                return '
      ' + code + '
      ';
-            }
-            return '
      ' + code.replace(//g, '>') + '
      ';
+    // Configure marked
+    if (typeof marked !== 'undefined') {
+        marked.setOptions({
+            gfm: true,
+            breaks: true
         });
-        
-        // Inline code
-        html = html.replace(/`([^`]+)`/g, '$1');
-        
-        // Links
-        html = html.replace(/\[([^\]]+)\]\(([^\)]+)\)/g, '$1');
-        
-        // Images
-        html = html.replace(/!\[([^\]]*)\]\(([^\)]+)\)/g, '$1');
-        
-        // Blockquotes
-        html = html.replace(/^\> (.+)$/gim, '
      $1
      ');
-        
-        // Horizontal rules
-        html = html.replace(/^---$/gim, '
      ');
-        html = html.replace(/^\*\*\*$/gim, '
      ');
-        
-        // Unordered lists
-        html = html.replace(/^\* (.+)$/gim, '
    • $1
      • ');
-        html = html.replace(/^- (.+)$/gim, '
    • $1
      • ');
-        html = html.replace(/(
    • .*<\/li>)/s, '
        $1
      ');
-        
-        // Ordered lists
-        html = html.replace(/^\d+\. (.+)$/gim, '
    • $1
      • ');
-        
-        // Tables (basic support)
-        html = html.replace(/\|(.+)\|/g, function(match) {
-            const cells = match.split('|').filter(cell => cell.trim());
-            return '
    ' + cells.map(cell => '').join('') + ''; - }); - html = html.replace(/(.*<\/tr>)/s, '
    ' + cell.trim() + '
    $1
    '); - - // Paragraphs - html = html.replace(/\n\n/g, '

    '); - html = '

    ' + html + '

    '; - - // Clean up - html = html.replace(/

    <\/p>/g, ''); - html = html.replace(/

    ()/g, '$1'); - html = html.replace(/(<\/h[1-6]>)<\/p>/g, '$1'); - html = html.replace(/

    ()/g, '$1'); - html = html.replace(/(<\/table>)<\/p>/g, '$1'); - html = html.replace(/

    (

      )/g, '$1'); - html = html.replace(/(<\/ul>)<\/p>/g, '$1'); - html = html.replace(/

      (

      )/g, '$1');
-        html = html.replace(/(<\/pre>)<\/p>/g, '$1');
-        html = html.replace(/
      (
      )/g, '$1');
-        html = html.replace(/(<\/div>)<\/p>/g, '$1');
-        html = html.replace(/
      (
      )<\/p>/g, '$1');
-        html = html.replace(/
      (
      )/g, '$1');
-        html = html.replace(/(<\/blockquote>)<\/p>/g, '$1');
-        
-        return html;
     }
     
     // Initialize mermaid
@@ -181,11 +104,22 @@
         }
         
         async renderMarkdown(markdown) {
-            const html = parseMarkdown(markdown);
             const content = document.getElementById('markdown-content');
+            
+            // Parse markdown to HTML
+            const html = marked.parse(markdown);
             content.innerHTML = html;
             
-            // Process mermaid diagrams
+            // Convert mermaid code blocks to mermaid divs
+            const mermaidBlocks = content.querySelectorAll('code.language-mermaid');
+            mermaidBlocks.forEach((block) => {
+                const div = document.createElement('div');
+                div.className = 'mermaid';
+                div.textContent = block.textContent;
+                block.parentElement.replaceWith(div);
+            });
+            
+            // Render mermaid diagrams
             const mermaidDivs = content.querySelectorAll('.mermaid');
             if (mermaidDivs.length > 0) {
                 try {
@@ -202,47 +136,33 @@
         }
     }
     
-    // Navigation tree renderer with proper subsection support
+    // Navigation tree renderer
     function renderNavTree(tree, container, depth = 0) {
         if (!tree || typeof tree !== 'object') return;
         
         for (const [moduleName, moduleData] of Object.entries(tree)) {
+            // Skip metadata fields
             if (moduleName === 'description' || moduleName === 'components') continue;
             
-            // Check if this has components (is a page) or just children (is a section)
-            const hasComponents = moduleData && moduleData.components && moduleData.components.length > 0;
-            const hasChildren = moduleData && moduleData.children && Object.keys(moduleData.children).length > 0;
-            
-            if (hasComponents) {
-                // This is a page - render as link
-                const item = document.createElement('div');
-                item.className = depth > 0 ? 'nav-subsection' : 'nav-item';
-                
-                const link = document.createElement('a');
-                link.href = `#/${moduleName}.md`;
-                link.textContent = formatModuleName(moduleName);
-                link.className = 'nav-link';
-                
-                item.appendChild(link);
-                container.appendChild(item);
-            } else if (hasChildren) {
-                // This is a section header - render as text
-                const sectionHeader = document.createElement('div');
-                sectionHeader.className = 'nav-section-header';
-                sectionHeader.textContent = formatModuleName(moduleName);
-                if (depth > 0) {
-                    sectionHeader.style.fontSize = `${14 - (depth * 1)}px`;
-                    sectionHeader.style.textTransform = 'none';
-                }
-                container.appendChild(sectionHeader);
+            // Create navigation item
+            const item = document.createElement('div');
+            if (depth > 0) {
+                item.className = 'nav-subsection';
+            } else {
+                item.className = 'nav-item';
             }
             
-            // Render children recursively
-            if (hasChildren) {
-                const subsection = document.createElement('div');
-                subsection.className = 'nav-subsection';
-                container.appendChild(subsection);
-                renderNavTree(moduleData.children, subsection, depth + 1);
+            const link = document.createElement('a');
+            link.href = `#/${moduleName}.md`;
+            link.textContent = formatModuleName(moduleName);
+            link.className = 'nav-link';
+            
+            item.appendChild(link);
+            container.appendChild(item);
+            
+            // Render children recursively if they exist
+            if (moduleData && moduleData.children && Object.keys(moduleData.children).length > 0) {
+                renderNavTree(moduleData.children, container, depth + 1);
             }
         }
     }
diff --git a/codewiki/templates/github_pages/index.html b/codewiki/templates/github_pages/index.html
index c711b30e..8dfbca84 100644
--- a/codewiki/templates/github_pages/index.html
+++ b/codewiki/templates/github_pages/index.html
@@ -7,6 +7,7 @@
     
     
     
+    
     
     
     
+
+
+    
      
+        
+        
+        
      
+            
      
+                
      Loading...
      
+            
      
+        
      
+    
      
+    
+    
+
+'''
+        
+        return html
+    
+    def _escape_html(self, text: str) -> str:
+        """
+        Escape HTML special characters.
+        
+        Args:
+            text: Text to escape
+            
+        Returns:
+            Escaped text
+        """
+        if not text:
+            return ""
+        
+        return (text
+            .replace("&", "&")
+            .replace("<", "<")
+            .replace(">", ">")
+            .replace('"', """)
+            .replace("'", "'"))
+
     
     def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]:
         """
diff --git a/codewiki/templates/github_pages/app.js b/codewiki/templates/github_pages/app.js
deleted file mode 100644
index a01deb6f..00000000
--- a/codewiki/templates/github_pages/app.js
+++ /dev/null
@@ -1,276 +0,0 @@
-// CodeWiki GitHub Pages Viewer Application
-
-(function() {
-    'use strict';
-    
-    // Configure marked
-    if (typeof marked !== 'undefined') {
-        marked.setOptions({
-            gfm: true,
-            breaks: true
-        });
-    }
-    
-    // Initialize mermaid
-    mermaid.initialize({
-        startOnLoad: false,
-        theme: 'default',
-        themeVariables: {
-            primaryColor: '#2563eb',
-            primaryTextColor: '#334155',
-            primaryBorderColor: '#e2e8f0',
-            lineColor: '#64748b',
-            sectionBkgColor: '#f8fafc',
-            altSectionBkgColor: '#f1f5f9',
-            gridColor: '#e2e8f0',
-            secondaryColor: '#f1f5f9',
-            tertiaryColor: '#f8fafc'
-        },
-        flowchart: {
-            htmlLabels: true,
-            curve: 'basis'
-        },
-        sequence: {
-            diagramMarginX: 50,
-            diagramMarginY: 10,
-            actorMargin: 50,
-            width: 150,
-            height: 65,
-            boxMargin: 10,
-            boxTextMargin: 5,
-            noteMargin: 10,
-            messageMargin: 35,
-            mirrorActors: true,
-            bottomMarginAdj: 1,
-            useMaxWidth: true,
-            rightAngles: false,
-            showSequenceNumbers: false
-        }
-    });
-    
-    // Router
-    class Router {
-        constructor() {
-            this.routes = {};
-            this.currentRoute = null;
-            window.addEventListener('hashchange', () => this.handleRoute());
-        }
-        
-        handleRoute() {
-            const hash = window.location.hash.slice(1) || '/';
-            const cleanHash = hash.startsWith('/') ? hash.slice(1) : hash;
-            const path = (hash === '/' || cleanHash === '') ? 'overview.md' : cleanHash;
-            this.loadMarkdown(path);
-        }
-        
-        async loadMarkdown(filename) {
-            const content = document.getElementById('markdown-content');
-            
-            // Show loading state
-            content.innerHTML = '
      Loading...
      ';
-            
-            try {
-                const response = await fetch(filename);
-                if (!response.ok) {
-                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-                }
-                const markdown = await response.text();
-                this.renderMarkdown(markdown);
-                this.updateActiveLink(filename);
-            } catch (error) {
-                console.error('Error loading markdown:', error);
-                content.innerHTML = `
-                    
      📄 Document Not Found
      
-                    
      Could not load ${filename}
      
-                    
      
-                        ${error.message}
-                    
      
-                    
      
-                        ← Back to Overview
-                    
      
-                `;
-            }
-        }
-        
-        updateActiveLink(filename) {
-            document.querySelectorAll('.nav-link').forEach(link => {
-                link.classList.remove('active');
-                const linkPath = link.getAttribute('href').replace('#/', '').replace('#', '');
-                const currentPath = filename;
-                if (linkPath === currentPath || (linkPath === '' && filename === 'overview.md')) {
-                    link.classList.add('active');
-                }
-            });
-        }
-        
-        async renderMarkdown(markdown) {
-            const content = document.getElementById('markdown-content');
-            
-            // Parse markdown to HTML
-            const html = marked.parse(markdown);
-            content.innerHTML = html;
-            
-            // Convert mermaid code blocks to mermaid divs
-            const mermaidBlocks = content.querySelectorAll('code.language-mermaid');
-            mermaidBlocks.forEach((block) => {
-                const div = document.createElement('div');
-                div.className = 'mermaid';
-                div.textContent = block.textContent;
-                block.parentElement.replaceWith(div);
-            });
-            
-            // Render mermaid diagrams
-            const mermaidDivs = content.querySelectorAll('.mermaid');
-            if (mermaidDivs.length > 0) {
-                try {
-                    await mermaid.run({
-                        nodes: mermaidDivs
-                    });
-                } catch (e) {
-                    console.error('Mermaid rendering error:', e);
-                }
-            }
-            
-            // Scroll to top
-            window.scrollTo(0, 0);
-        }
-    }
-    
-    // Navigation tree renderer
-    function renderNavTree(tree, container, depth = 0) {
-        if (!tree || typeof tree !== 'object') return;
-        
-        for (const [moduleName, moduleData] of Object.entries(tree)) {
-            // Skip metadata fields
-            if (moduleName === 'description' || moduleName === 'components') continue;
-            
-            // Create navigation item
-            const item = document.createElement('div');
-            if (depth > 0) {
-                item.className = 'nav-subsection';
-            } else {
-                item.className = 'nav-item';
-            }
-            
-            const link = document.createElement('a');
-            link.href = `#/${moduleName}.md`;
-            link.textContent = formatModuleName(moduleName);
-            link.className = 'nav-link';
-            
-            item.appendChild(link);
-            container.appendChild(item);
-            
-            // Render children recursively if they exist
-            if (moduleData && moduleData.children && Object.keys(moduleData.children).length > 0) {
-                renderNavTree(moduleData.children, container, depth + 1);
-            }
-        }
-    }
-    
-    // Format module name for display
-    function formatModuleName(name) {
-        return name
-            .replace(/[_-]/g, ' ')
-            .split(' ')
-            .map(word => word.charAt(0).toUpperCase() + word.slice(1))
-            .join(' ');
-    }
-    
-    // Render metadata info box
-    function renderMetadata(metadata) {
-        if (!metadata || !metadata.generation_info) return null;
-        
-        const metadataBox = document.createElement('div');
-        metadataBox.style.cssText = `
-            margin: 20px 0;
-            padding: 15px;
-            background: #f8fafc;
-            border-radius: 8px;
-            border: 1px solid #e2e8f0;
-        `;
-        
-        const header = document.createElement('h4');
-        header.textContent = 'GENERATION INFO';
-        header.style.cssText = `
-            margin: 0 0 10px 0;
-            font-size: 12px;
-            color: #64748b;
-            text-transform: uppercase;
-            letter-spacing: 0.05em;
-        `;
-        metadataBox.appendChild(header);
-        
-        const info = metadata.generation_info;
-        const contentDiv = document.createElement('div');
-        contentDiv.style.cssText = 'font-size: 11px; color: #475569; line-height: 1.4;';
-        
-        let html = '';
-        if (info.main_model) {
-            html += `
      Model: ${info.main_model}
      `;
-        }
-        if (info.timestamp) {
-            const date = new Date(info.timestamp);
-            html += `
      Generated: ${date.toLocaleString().substring(0, 16)}
      `;
-        }
-        if (info.commit_id) {
-            html += `
      Commit: ${info.commit_id.substring(0, 8)}
      `;
-        }
-        if (metadata.statistics && metadata.statistics.total_components) {
-            html += `
      Components: ${metadata.statistics.total_components}
      `;
-        }
-        
-        contentDiv.innerHTML = html;
-        metadataBox.appendChild(contentDiv);
-        
-        return metadataBox;
-    }
-    
-    // Initialize application
-    function init() {
-        const navTree = document.getElementById('nav-tree');
-        
-        if (MODULE_TREE) {
-            // Add overview link
-            const navSection = document.createElement('div');
-            navSection.className = 'nav-section';
-            
-            const overviewItem = document.createElement('div');
-            overviewItem.className = 'nav-item';
-            const overviewLink = document.createElement('a');
-            overviewLink.href = '#/';
-            overviewLink.textContent = 'Overview';
-            overviewLink.className = 'nav-link';
-            overviewItem.appendChild(overviewLink);
-            navSection.appendChild(overviewItem);
-            navTree.appendChild(navSection);
-            
-            // Add metadata info box if available
-            if (CODEWIKI_CONFIG && CODEWIKI_CONFIG.metadata) {
-                const metadataBox = renderMetadata(CODEWIKI_CONFIG.metadata);
-                if (metadataBox) {
-                    document.getElementById('metadata-info').appendChild(metadataBox);
-                }
-            }
-            
-            // Render module tree
-            const moduleSection = document.createElement('div');
-            moduleSection.className = 'nav-section';
-            navTree.appendChild(moduleSection);
-            renderNavTree(MODULE_TREE, moduleSection);
-        }
-        
-        // Initialize router
-        const router = new Router();
-        
-        // Initial load
-        router.handleRoute();
-    }
-    
-    // Start application when DOM is ready
-    if (document.readyState === 'loading') {
-        document.addEventListener('DOMContentLoaded', init);
-    } else {
-        init();
-    }
-})();
-
diff --git a/codewiki/templates/github_pages/index.html b/codewiki/templates/github_pages/index.html
deleted file mode 100644
index 8dfbca84..00000000
--- a/codewiki/templates/github_pages/index.html
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
-    
-    
-    {{TITLE}}
-    
-    
-    
-    
-    
-    
-    
-
-
-    
      
-        
-        
-        
      
-            
      
-                
      Loading...
      
-            
      
-        
      
-    
      
-
-    
-    
-
-    
-    
-
-
-
diff --git a/codewiki/templates/github_pages/styles.css b/codewiki/templates/github_pages/styles.css
deleted file mode 100644
index c2ef7da6..00000000
--- a/codewiki/templates/github_pages/styles.css
+++ /dev/null
@@ -1,272 +0,0 @@
-:root {
-    --primary-color: #2563eb;
-    --secondary-color: #f1f5f9;
-    --text-color: #334155;
-    --border-color: #e2e8f0;
-    --hover-color: #f8fafc;
-}
-
-* {
-    margin: 0;
-    padding: 0;
-    box-sizing: border-box;
-}
-
-body {
-    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
-    line-height: 1.6;
-    color: var(--text-color);
-    background-color: #ffffff;
-}
-
-.container {
-    display: flex;
-    min-height: 100vh;
-}
-
-.sidebar {
-    width: 300px;
-    background-color: var(--secondary-color);
-    border-right: 1px solid var(--border-color);
-    padding: 20px;
-    overflow-y: auto;
-    position: fixed;
-    height: 100vh;
-}
-
-.content {
-    flex: 1;
-    margin-left: 300px;
-    padding: 40px 60px;
-    max-width: calc(100% - 300px);
-}
-
-.logo {
-    font-size: 24px;
-    font-weight: bold;
-    color: var(--primary-color);
-    margin-bottom: 30px;
-    text-decoration: none;
-    display: block;
-}
-
-.nav-section {
-    margin-bottom: 25px;
-}
-
-.nav-section h3 {
-    font-size: 14px;
-    font-weight: 600;
-    color: #64748b;
-    text-transform: uppercase;
-    letter-spacing: 0.05em;
-    margin-bottom: 10px;
-}
-
-.nav-item {
-    display: block;
-    margin-bottom: 2px;
-}
-
-.nav-link {
-    display: block;
-    padding: 8px 12px;
-    color: var(--text-color);
-    text-decoration: none;
-    border-radius: 6px;
-    font-size: 14px;
-    transition: all 0.2s ease;
-}
-
-.nav-link:hover {
-    background-color: var(--hover-color);
-    color: var(--primary-color);
-}
-
-.nav-link.active {
-    background-color: var(--primary-color);
-    color: white;
-}
-
-.nav-subsection {
-    margin-left: 15px;
-    margin-top: 8px;
-}
-
-.nav-subsection .nav-link {
-    font-size: 13px;
-    color: #64748b;
-}
-
-.nav-section-header {
-    font-size: 14px;
-    font-weight: 600;
-    color: #64748b;
-    text-transform: uppercase;
-    letter-spacing: 0.05em;
-    margin-bottom: 10px;
-    padding: 8px 12px;
-}
-
-/* Nested subsection indentation - scalable for any depth */
-.nav-subsection .nav-subsection {
-    margin-left: 20px;
-}
-
-.nav-subsection .nav-subsection .nav-link {
-    font-size: 12px;
-}
-
-/* Additional nesting levels */
-.nav-subsection .nav-subsection .nav-subsection {
-    margin-left: 15px;
-}
-
-.nav-subsection .nav-subsection .nav-subsection .nav-link {
-    font-size: 11px;
-}
-
-.markdown-content {
-    max-width: none;
-}
-
-.markdown-content h1 {
-    font-size: 2.5rem;
-    font-weight: 700;
-    color: #1e293b;
-    margin-bottom: 1rem;
-    border-bottom: 2px solid var(--border-color);
-    padding-bottom: 0.5rem;
-}
-
-.markdown-content h2 {
-    font-size: 2rem;
-    font-weight: 600;
-    color: #334155;
-    margin-top: 2rem;
-    margin-bottom: 1rem;
-}
-
-.markdown-content h3 {
-    font-size: 1.5rem;
-    font-weight: 600;
-    color: #475569;
-    margin-top: 1.5rem;
-    margin-bottom: 0.75rem;
-}
-
-.markdown-content p {
-    margin-bottom: 1rem;
-    color: #475569;
-}
-
-.markdown-content ul, .markdown-content ol {
-    margin-bottom: 1rem;
-    padding-left: 1.5rem;
-}
-
-.markdown-content li {
-    margin-bottom: 0.5rem;
-    color: #475569;
-}
-
-.markdown-content code {
-    background-color: #f1f5f9;
-    padding: 0.25rem 0.5rem;
-    border-radius: 0.25rem;
-    font-family: 'Fira Code', 'Consolas', monospace;
-    font-size: 0.875rem;
-}
-
-.markdown-content pre {
-    background-color: #f8fafc;
-    border: 1px solid var(--border-color);
-    border-radius: 0.5rem;
-    padding: 1rem;
-    overflow-x: auto;
-    margin-bottom: 1rem;
-}
-
-.markdown-content pre code {
-    background-color: transparent;
-    padding: 0;
-}
-
-.markdown-content blockquote {
-    border-left: 4px solid var(--primary-color);
-    padding-left: 1rem;
-    margin-bottom: 1rem;
-    font-style: italic;
-    color: #64748b;
-}
-
-.markdown-content table {
-    width: 100%;
-    border-collapse: collapse;
-    margin-bottom: 1rem;
-}
-
-.markdown-content th, .markdown-content td {
-    border: 1px solid var(--border-color);
-    padding: 0.75rem;
-    text-align: left;
-}
-
-.markdown-content th {
-    background-color: var(--secondary-color);
-    font-weight: 600;
-}
-
-.markdown-content a {
-    color: var(--primary-color);
-    text-decoration: underline;
-}
-
-.markdown-content a:hover {
-    text-decoration: none;
-}
-
-.markdown-content img {
-    max-width: 100%;
-    height: auto;
-    border-radius: 0.5rem;
-    margin: 1rem 0;
-}
-
-.markdown-content hr {
-    border: none;
-    border-top: 1px solid var(--border-color);
-    margin: 2rem 0;
-}
-
-/* Mermaid diagrams */
-.mermaid {
-    background: #ffffff;
-    border: 1px solid var(--border-color);
-    border-radius: 6px;
-    padding: 1rem;
-    margin: 1rem 0;
-    text-align: center;
-}
-
-/* Loading state */
-.loading {
-    text-align: center;
-    padding: 2rem;
-    color: #64748b;
-}
-
-@media (max-width: 768px) {
-    .sidebar {
-        width: 100%;
-        position: relative;
-        height: auto;
-    }
-    
-    .content {
-        margin-left: 0;
-        padding: 20px;
-        max-width: 100%;
-    }
-}
-

From be685f25f2411025ce0e04af0834f70e4f5256e5 Mon Sep 17 00:00:00 2001
From: anhnh2002 
Date: Sat, 18 Oct 2025 23:54:14 +0700
Subject: [PATCH 15/17] adjust docs UI

---
 codewiki/cli/html_generator.py | 163 +++++++++++++++++++++++++++------
 1 file changed, 137 insertions(+), 26 deletions(-)

diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py
index efc17a01..ad610269 100644
--- a/codewiki/cli/html_generator.py
+++ b/codewiki/cli/html_generator.py
@@ -81,6 +81,32 @@ def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]:
             return None
     
     
+    def load_all_markdown_files(self, docs_dir: Path) -> Dict[str, str]:
+        """
+        Load all markdown files from documentation directory.
+        
+        Args:
+            docs_dir: Documentation directory path
+            
+        Returns:
+            Dictionary mapping filename to markdown content
+        """
+        markdown_files = {}
+        
+        if not docs_dir.exists():
+            return markdown_files
+        
+        # Load all .md files in the directory
+        for md_file in docs_dir.glob("*.md"):
+            try:
+                content = safe_read(md_file)
+                markdown_files[md_file.name] = content
+            except Exception:
+                # Skip files that can't be read
+                pass
+        
+        return markdown_files
+    
     def generate(
         self,
         output_path: Path,
@@ -112,6 +138,11 @@ def generate(
         if docs_dir and metadata is None:
             metadata = self.load_metadata(docs_dir)
         
+        # Load all markdown content
+        markdown_content = {}
+        if docs_dir:
+            markdown_content = self.load_all_markdown_files(docs_dir)
+        
         # Default values
         if module_tree is None:
             module_tree = {}
@@ -126,7 +157,8 @@ def generate(
             repository_url=repository_url,
             github_pages_url=github_pages_url,
             metadata=metadata,
-            config=config
+            config=config,
+            markdown_content=markdown_content
         )
         
         # Write to file
@@ -139,7 +171,8 @@ def _generate_html_template(
         repository_url: Optional[str],
         github_pages_url: Optional[str],
         metadata: Optional[Dict[str, Any]],
-        config: Dict[str, Any]
+        config: Dict[str, Any],
+        markdown_content: Dict[str, str] = None
     ) -> str:
         """
         Generate the complete HTML template with embedded styles and scripts.
@@ -151,6 +184,7 @@ def _generate_html_template(
             github_pages_url: GitHub Pages URL
             metadata: Metadata dictionary
             config: Additional configuration
+            markdown_content: Dictionary of markdown file contents
             
         Returns:
             Complete HTML string
@@ -158,6 +192,7 @@ def _generate_html_template(
         # Escape and prepare JSON data for embedding
         module_tree_json = json.dumps(module_tree)
         metadata_json = json.dumps(metadata) if metadata else "null"
+        markdown_content_json = json.dumps(markdown_content) if markdown_content else "{}"
         
         html = f'''
 
@@ -448,7 +483,7 @@ def _generate_html_template(
 
     
      
         
      ';
+                contentEl.innerHTML = errorHtml;
             }}
         }}
         
@@ -672,24 +737,70 @@ def _generate_html_template(
         
         // Initialize app
         async function init() {{
-            // Render metadata and navigation
-            renderMetadata();
-            renderNavigation();
+            console.log('[init] Starting initialization');
+            console.log('[init] marked:', typeof marked);
+            console.log('[init] mermaid:', typeof mermaid);
+            console.log('[init] MARKDOWN_CONTENT keys:', Object.keys(MARKDOWN_CONTENT || {{}}).slice(0, 5));
             
-            // Load initial page from hash or default to overview
-            const hash = window.location.hash.substring(1);
-            const initialPage = hash && hash.endsWith('.md') ? hash : 'overview.md';
-            await loadPage(initialPage);
-            
-            // Listen for hash changes
-            window.addEventListener('hashchange', handleHashChange);
+            try {{
+                // Render metadata and navigation
+                console.log('[init] Rendering metadata');
+                renderMetadata();
+                console.log('[init] Rendering navigation');
+                renderNavigation();
+                
+                // Load initial page from hash or default to overview
+                const hash = window.location.hash.substring(1);
+                const initialPage = hash && hash.endsWith('.md') ? hash : 'overview.md';
+                console.log('[init] Initial page:', initialPage);
+                await loadPage(initialPage);
+                console.log('[init] Page loaded successfully');
+                
+                // Listen for hash changes
+                window.addEventListener('hashchange', handleHashChange);
+                console.log('[init] Initialization complete');
+            }} catch (error) {{
+                console.error('[init] ERROR:', error);
+                const contentEl = document.getElementById('content');
+                contentEl.innerHTML = `
+                    
      
+                        
      Initialization Error
      
+                        
      Error: ${{error.message}}
      
+                        
      ${{error.stack}}
      
+                    
      
+                `;
+            }}
+        }}
+        
+        // Check if external libraries loaded
+        function checkLibraries() {{
+            if (typeof marked === 'undefined' || typeof mermaid === 'undefined') {{
+                const contentEl = document.getElementById('content');
+                contentEl.innerHTML = `
+                    
      
+                        
      External Libraries Not Loaded
      
+                        
      The required JavaScript libraries (marked.js and mermaid.js) failed to load.
      
+                        
      This usually happens when opening the HTML file directly (file:// protocol).
      
+                        
      Solution:
      
+                        
      Serve the documentation with a local web server:
      
+                        
        
+                            
      1. Python: python3 -m http.server 8000 then visit http://localhost:8000
        2. 
+                            
      2. Or deploy to GitHub Pages (works perfectly there!)
        3. 
+                        
      
+                    
      
+                `;
+                return false;
+            }}
+            return true;
         }}
         
         // Start when DOM is ready
         if (document.readyState === 'loading') {{
-            document.addEventListener('DOMContentLoaded', init);
+            document.addEventListener('DOMContentLoaded', () => {{
+                if (checkLibraries()) init();
+            }});
         }} else {{
-            init();
+            if (checkLibraries()) init();
         }}
     
 

From 465cc9214ae6c4c54236f0416be26413d58382e7 Mon Sep 17 00:00:00 2001
From: anhnh2002 
Date: Sun, 19 Oct 2025 10:22:41 +0700
Subject: [PATCH 16/17] fix UI

---
 codewiki/cli/html_generator.py                | 790 +++---------------
 codewiki/cli/utils/instructions.py            |   6 +-
 .../github_pages/viewer_template.html         | 643 ++++++++++++++
 3 files changed, 744 insertions(+), 695 deletions(-)
 create mode 100644 codewiki/templates/github_pages/viewer_template.html

diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py
index ad610269..817c77e7 100644
--- a/codewiki/cli/html_generator.py
+++ b/codewiki/cli/html_generator.py
@@ -79,34 +79,7 @@ def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]:
         except Exception:
             # Non-critical, return None
             return None
-    
-    
-    def load_all_markdown_files(self, docs_dir: Path) -> Dict[str, str]:
-        """
-        Load all markdown files from documentation directory.
-        
-        Args:
-            docs_dir: Documentation directory path
             
-        Returns:
-            Dictionary mapping filename to markdown content
-        """
-        markdown_files = {}
-        
-        if not docs_dir.exists():
-            return markdown_files
-        
-        # Load all .md files in the directory
-        for md_file in docs_dir.glob("*.md"):
-            try:
-                content = safe_read(md_file)
-                markdown_files[md_file.name] = content
-            except Exception:
-                # Skip files that can't be read
-                pass
-        
-        return markdown_files
-    
     def generate(
         self,
         output_path: Path,
@@ -131,682 +104,117 @@ def generate(
             docs_dir: Documentation directory (for auto-loading module_tree and metadata)
             metadata: Metadata dictionary (auto-loaded from docs_dir if not provided)
         """
-        # Auto-load module tree and metadata if docs_dir provided
-        if docs_dir and module_tree is None:
-            module_tree = self.load_module_tree(docs_dir)
-        
-        if docs_dir and metadata is None:
-            metadata = self.load_metadata(docs_dir)
-        
-        # Load all markdown content
-        markdown_content = {}
+        # Auto-load module_tree and metadata from docs_dir if not provided
         if docs_dir:
-            markdown_content = self.load_all_markdown_files(docs_dir)
+            if module_tree is None:
+                module_tree = self.load_module_tree(docs_dir)
+            if metadata is None:
+                metadata = self.load_metadata(docs_dir)
         
         # Default values
         if module_tree is None:
             module_tree = {}
-        
         if config is None:
             config = {}
         
-        # Generate HTML content
-        html_content = self._generate_html_template(
-            title=title,
-            module_tree=module_tree,
-            repository_url=repository_url,
-            github_pages_url=github_pages_url,
-            metadata=metadata,
-            config=config,
-            markdown_content=markdown_content
-        )
-        
-        # Write to file
+        # Load template
+        template_path = self.template_dir / "viewer_template.html"
+        if not template_path.exists():
+            raise FileSystemError(f"Template not found: {template_path}")
+        
+        template_content = safe_read(template_path)
+        
+        # Build info content HTML
+        info_content = self._build_info_content(metadata)
+        show_info = "block" if info_content else "none"
+        
+        # Build repository link
+        repo_link = ""
+        if repository_url:
+            repo_link = f'🔗 View Repository'
+        
+        # Determine docs base path
+        # For GitHub Pages: relative path to docs folder
+        # For local: relative path to docs folder
+        docs_base_path = ""
+        if docs_dir and output_path.parent != docs_dir:
+            # Calculate relative path from output to docs
+            try:
+                docs_base_path = Path(docs_dir.name).as_posix()
+            except Exception:
+                docs_base_path = "."
+        
+        # Prepare JSON data for embedding
+        config_json = json.dumps(config, indent=2)
+        module_tree_json = json.dumps(module_tree, indent=2)
+        metadata_json = json.dumps(metadata, indent=2) if metadata else "null"
+        
+        # Replace placeholders
+        html_content = template_content
+        replacements = {
+            "{{TITLE}}": self._escape_html(title),
+            "{{REPO_LINK}}": repo_link,
+            "{{SHOW_INFO}}": show_info,
+            "{{INFO_CONTENT}}": info_content,
+            "{{CONFIG_JSON}}": config_json,
+            "{{MODULE_TREE_JSON}}": module_tree_json,
+            "{{METADATA_JSON}}": metadata_json,
+            "{{DOCS_BASE_PATH}}": docs_base_path,
+        }
+        
+        for placeholder, value in replacements.items():
+            html_content = html_content.replace(placeholder, value)
+        
+        # Write output
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
         safe_write(output_path, html_content)
     
-    def _generate_html_template(
-        self,
-        title: str,
-        module_tree: Dict[str, Any],
-        repository_url: Optional[str],
-        github_pages_url: Optional[str],
-        metadata: Optional[Dict[str, Any]],
-        config: Dict[str, Any],
-        markdown_content: Dict[str, str] = None
-    ) -> str:
+    def _build_info_content(self, metadata: Optional[Dict[str, Any]]) -> str:
         """
-        Generate the complete HTML template with embedded styles and scripts.
+        Build HTML content for repo info section.
         
         Args:
-            title: Documentation title
-            module_tree: Module tree structure
-            repository_url: GitHub repository URL
-            github_pages_url: GitHub Pages URL
             metadata: Metadata dictionary
-            config: Additional configuration
-            markdown_content: Dictionary of markdown file contents
             
         Returns:
-            Complete HTML string
+            HTML string for info content
         """
-        # Escape and prepare JSON data for embedding
-        module_tree_json = json.dumps(module_tree)
-        metadata_json = json.dumps(metadata) if metadata else "null"
-        markdown_content_json = json.dumps(markdown_content) if markdown_content else "{}"
-        
-        html = f'''
-
-
-    
-    
-    {self._escape_html(title)}
-    
-    
-    
-    
-    
-    
-    
-    
-
-
-    
      
-        
-        
-        
      
-            
      
-                
      Loading...
      
-            
      
-        
      
-    
      
-    
-    
-
-'''
-        
-        return html
+        html_parts = []
+        
+        if info.get('main_model'):
+            html_parts.append(f'
      Model: {self._escape_html(info["main_model"])}
      ')
+        
+        if info.get('timestamp'):
+            try:
+                from datetime import datetime
+                timestamp = info['timestamp']
+                # Parse ISO format timestamp
+                if isinstance(timestamp, str):
+                    dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
+                    formatted_date = dt.strftime('%Y-%m-%d')
+                    html_parts.append(f'
      Generated: {formatted_date}
      ')
+            except Exception:
+                pass
+        
+        if info.get('commit_id'):
+            commit_short = info['commit_id'][:8]
+            html_parts.append(f'
      Commit: {commit_short}
      ')
+        
+        if stats.get('total_components'):
+            components_str = f"{stats['total_components']:,}"
+            html_parts.append(f'
      Components: {components_str}
      ')
+        
+        if stats.get('max_depth'):
+            html_parts.append(f'
      Max Depth: {stats["max_depth"]}
      ')
+        
+        return '\n                '.join(html_parts)
     
     def _escape_html(self, text: str) -> str:
         """
@@ -818,15 +226,13 @@ def _escape_html(self, text: str) -> str:
         Returns:
             Escaped text
         """
-        if not text:
-            return ""
-        
         return (text
-            .replace("&", "&")
-            .replace("<", "<")
-            .replace(">", ">")
-            .replace('"', """)
-            .replace("'", "'"))
+                .replace('&', '&')
+                .replace('<', '<')
+                .replace('>', '>')
+                .replace('"', '"')
+                .replace("'", '''))
+       
 
     
     def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]:
diff --git a/codewiki/cli/utils/instructions.py b/codewiki/cli/utils/instructions.py
index 11085d09..7c2bf058 100644
--- a/codewiki/cli/utils/instructions.py
+++ b/codewiki/cli/utils/instructions.py
@@ -95,9 +95,9 @@ def display_post_generation_instructions(
             minutes = int(statistics['generation_time'] // 60)
             seconds = int(statistics['generation_time'] % 60)
             click.echo(f"  Generation time:   {minutes} minutes {seconds} seconds")
-        if 'total_tokens_used' in statistics:
-            tokens = statistics['total_tokens_used']
-            click.echo(f"  Tokens used:       ~{tokens:,}")
+        # if 'total_tokens_used' in statistics:
+        #     tokens = statistics['total_tokens_used']
+        #     click.echo(f"  Tokens used:       ~{tokens:,}")
         click.echo()
     
     # Next steps
diff --git a/codewiki/templates/github_pages/viewer_template.html b/codewiki/templates/github_pages/viewer_template.html
new file mode 100644
index 00000000..39629ade
--- /dev/null
+++ b/codewiki/templates/github_pages/viewer_template.html
@@ -0,0 +1,643 @@
+
+
+
+    
+    
+    {{TITLE}}
+    
+    
+    
+
+
+    
      
+        
+        
+        
      
+            
      
+                
+                
      Loading documentation...
      
+            
      
+            
+            
+        
      
+    
      
+    
+    
+
+
+

From 655084a2709eff11206f7b5b6d3bbe73c0aa666b Mon Sep 17 00:00:00 2001
From: anhnh2002 
Date: Sun, 19 Oct 2025 10:37:46 +0700
Subject: [PATCH 17/17] re-organise repo structure and adjust readme

---
 DOCKER_README.md                              |  30 --
 HTMLGENERATOR_IMPROVEMENTS.md                 | 311 ------------
 README.md                                     | 304 ++++++++++--
 codewiki/README.md                            | 138 ------
 .dockerignore => docker/.dockerignore         |   0
 docker/DOCKER_README.md                       | 443 ++++++++++++++++++
 Dockerfile => docker/Dockerfile               |  11 +-
 .../docker-compose.yml                        |  10 +-
 env.example => docker/env.example             |   0
 9 files changed, 729 insertions(+), 518 deletions(-)
 delete mode 100644 DOCKER_README.md
 delete mode 100644 HTMLGENERATOR_IMPROVEMENTS.md
 delete mode 100644 codewiki/README.md
 rename .dockerignore => docker/.dockerignore (100%)
 create mode 100644 docker/DOCKER_README.md
 rename Dockerfile => docker/Dockerfile (77%)
 rename docker-compose.yml => docker/docker-compose.yml (84%)
 rename env.example => docker/env.example (100%)

diff --git a/DOCKER_README.md b/DOCKER_README.md
deleted file mode 100644
index 4aa968b6..00000000
--- a/DOCKER_README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# CodeWiki Docker Setup
-
-This document explains how to run CodeWiki using Docker and Docker Compose.
-
-## Quick Start
-
-1. **Clone the repository** (if not already done):
-   ```bash
-   git clone 
-   cd CodeWiki
-   ```
-
-2. **Set up environment variables**:
-   ```bash
-   cp env.example .env
-   # Edit .env file with your API keys
-   ```
-
-3. **Create network**
-   ```bash
-   docker network create codewiki-network
-   ```
-
-3. **Start the services**:
-   ```bash
-   docker-compose up -d
-   ```
-
-4. **Access the application**:
-   - Main web app: http://localhost:8000
diff --git a/HTMLGENERATOR_IMPROVEMENTS.md b/HTMLGENERATOR_IMPROVEMENTS.md
deleted file mode 100644
index 91f757ed..00000000
--- a/HTMLGENERATOR_IMPROVEMENTS.md
+++ /dev/null
@@ -1,311 +0,0 @@
-# HTMLGenerator and Template Improvements
-
-## Summary
-
-Updated the HTMLGenerator class and its associated templates based on the frontend code and generated documentation examples. The improvements enable automatic loading of module trees and metadata, better navigation rendering, and enhanced user experience.
-
-## Changes Made
-
-### 1. HTMLGenerator Class (`codewiki/cli/html_generator.py`)
-
-#### New Methods
-
-**`load_module_tree(docs_dir: Path)`**
-- Automatically loads `module_tree.json` from the documentation directory
-- Provides fallback structure if file doesn't exist
-- Returns properly structured module tree for navigation rendering
-
-**`load_metadata(docs_dir: Path)`**
-- Loads `metadata.json` from the documentation directory
-- Returns None if not found (non-critical)
-- Enables display of generation information in the viewer
-
-#### Updated `generate()` Method
-
-**New Parameters:**
-- `docs_dir`: Documentation directory path for auto-loading
-- `metadata`: Metadata dictionary (auto-loaded if docs_dir provided)
-- `module_tree`: Now optional, auto-loaded from docs_dir if not provided
-
-**Key Improvements:**
-- Auto-loads module_tree.json and metadata.json when docs_dir is provided
-- Embeds metadata in the configuration for client-side display
-- Maintains backward compatibility with existing code
-
-**Before:**
-```python
-html_generator.generate(
-    output_path=output_path,
-    title=title,
-    module_tree=manual_module_tree,  # Had to be manually provided
-    repository_url=url
-)
-```
-
-**After:**
-```python
-html_generator.generate(
-    output_path=output_path,
-    title=title,
-    docs_dir=output_dir,  # Auto-loads module_tree and metadata
-    repository_url=url
-)
-```
-
-### 2. Documentation Generator Adapter (`codewiki/cli/adapters/doc_generator.py`)
-
-#### Updated `_run_html_generation()` Method
-
-**Before:**
-- Created placeholder module tree with just "Overview"
-- Ignored the actual generated module structure
-- No metadata integration
-
-**After:**
-- Uses `docs_dir` parameter for automatic loading
-- Properly loads the module_tree.json generated in stage 2
-- Automatically loads metadata.json for display
-- Adds progress update for loading phase
-
-### 3. JavaScript Application (`codewiki/templates/github_pages/app.js`)
-
-#### Enhanced Navigation Rendering
-
-**`renderNavTree()` Function:**
-- Improved recursive rendering for deeply nested structures
-- Skips metadata properties ('description', 'components') in iteration
-- Better handling of module data objects
-- Properly renders children at any depth level
-
-**New `formatModuleName()` Function:**
-- Converts underscores and hyphens to spaces
-- Capitalizes each word for better readability
-- Example: `search_providers` → `Search Providers`
-
-**New `renderMetadata()` Function:**
-- Creates styled metadata info box for sidebar
-- Displays generation information:
-  - Model used for generation
-  - Timestamp (formatted as locale string)
-  - Commit ID (first 8 characters)
-  - Total components count
-- Returns null if no metadata available
-
-#### Updated `init()` Function
-
-**Additions:**
-- Renders metadata info box in sidebar after overview link
-- Positioned between overview and divider for prominence
-- Only displays if metadata is available in config
-
-### 4. Styles (`codewiki/templates/github_pages/styles.css`)
-
-#### Navigation Improvements
-
-**Font Sizing for Nested Levels:**
-- Base level: 0.875rem
-- Second level (2rem padding): 0.8125rem, subdued color
-- Third level (3rem padding): 0.75rem, more subdued color
-- Improves visual hierarchy for deep structures
-
-**New Metadata Box Styling:**
-- Dedicated `.metadata-box` class
-- Consistent styling with theme
-- Proper spacing and borders
-
-#### Content Enhancements
-
-**Blockquote Styling:**
-- Blue left border (matches theme)
-- Light background
-- Proper padding and margins
-
-**Image Handling:**
-- `max-width: 100%` for responsiveness
-- Auto height to maintain aspect ratio
-- Rounded corners (6px)
-- Proper margins
-
-**Horizontal Rules:**
-- Clean, minimal styling
-- Consistent with theme colors
-- Generous vertical spacing
-
-**Mermaid Diagrams:**
-- White background for contrast
-- Light border and rounded corners
-- Center alignment
-- Proper padding and margins
-
-**Loading State:**
-- Center-aligned loading indicator
-- Subdued color scheme
-- Adequate padding
-
-#### Layout Improvements
-
-**Footer Support:**
-- `#app` uses `min-height` instead of `height` for proper scrolling
-- `#footer` uses `margin-top: auto` to stick to bottom
-- Proper flex layout for sticky footer
-
-### 5. HTML Template (`codewiki/templates/github_pages/index.html`)
-
-#### New Footer Section
-
-**Added:**
-- Footer with CodeWiki attribution
-- Positioned at bottom of page
-- Subtle styling with theme colors
-- Link to project (placeholder for actual URL)
-
-**Benefits:**
-- Professional appearance
-- Clear attribution
-- Consistent with modern web practices
-
-## Benefits of These Improvements
-
-### 1. Automated Data Loading
-- No need to manually load module_tree.json
-- Metadata automatically integrated
-- Reduces code duplication
-- Fewer errors from manual data handling
-
-### 2. Better Navigation
-- Proper rendering of nested module structures
-- Clear visual hierarchy
-- Better readability with formatted names
-- Supports arbitrary nesting depth
-
-### 3. Enhanced User Experience
-- Generation information visible in sidebar
-- Model, timestamp, commit info at a glance
-- Better understanding of documentation provenance
-- Professional appearance with footer
-
-### 4. Improved Styling
-- Better support for nested navigation
-- Enhanced markdown rendering
-- Responsive images
-- Proper mermaid diagram display
-- Sticky footer layout
-
-### 5. Code Quality
-- Better separation of concerns
-- More reusable components
-- Improved error handling
-- Backward compatible
-
-## Testing Recommendations
-
-1. **Test with Real Repository:**
-   ```bash
-   cd /path/to/test-repo
-   codewiki generate --github-pages
-   ```
-
-2. **Verify Module Tree Rendering:**
-   - Check that all nested modules appear
-   - Verify indentation and hierarchy
-   - Test navigation clicks
-
-3. **Check Metadata Display:**
-   - Verify generation info box appears
-   - Check model name display
-   - Verify timestamp formatting
-   - Test with and without commit ID
-
-4. **Test Responsive Design:**
-   - View on mobile devices
-   - Test sidebar toggle
-   - Verify navigation usability
-   - Check content readability
-
-5. **Verify Markdown Rendering:**
-   - Test code blocks with syntax highlighting
-   - Verify mermaid diagram rendering
-   - Check image display
-   - Test blockquotes and lists
-
-## Migration Guide
-
-### For Existing Code
-
-**Old usage:**
-```python
-html_generator = HTMLGenerator()
-module_tree = load_some_module_tree()
-html_generator.generate(
-    output_path=output_path,
-    title=title,
-    module_tree=module_tree
-)
-```
-
-**New usage (recommended):**
-```python
-html_generator = HTMLGenerator()
-html_generator.generate(
-    output_path=output_path,
-    title=title,
-    docs_dir=docs_directory  # Auto-loads everything
-)
-```
-
-**Still supported (backward compatible):**
-```python
-html_generator = HTMLGenerator()
-html_generator.generate(
-    output_path=output_path,
-    title=title,
-    module_tree=manually_loaded_tree  # Still works
-)
-```
-
-## Future Enhancements
-
-Potential improvements for future iterations:
-
-1. **Search Functionality:**
-   - Full-text search across documentation
-   - Module/component search
-   - Code snippet search
-
-2. **Dark Mode:**
-   - Theme toggle
-   - System preference detection
-   - Persistent user preference
-
-3. **Table of Contents:**
-   - Auto-generated from headers
-   - Sticky sidebar TOC
-   - Smooth scrolling
-
-4. **Dependency Graphs:**
-   - Interactive visualization
-   - Load from temp/dependency_graphs
-   - Zoom and pan support
-
-5. **Code Copy Button:**
-   - One-click code copying
-   - Visual feedback
-   - Syntax-aware copying
-
-6. **Version Selector:**
-   - Multiple doc versions
-   - Comparison view
-   - Changelog integration
-
-## Files Modified
-
-1. `/Users/anhnh/Documents/vscode/CodeWiki/codewiki/cli/html_generator.py`
-2. `/Users/anhnh/Documents/vscode/CodeWiki/codewiki/cli/adapters/doc_generator.py`
-3. `/Users/anhnh/Documents/vscode/CodeWiki/codewiki/templates/github_pages/app.js`
-4. `/Users/anhnh/Documents/vscode/CodeWiki/codewiki/templates/github_pages/styles.css`
-5. `/Users/anhnh/Documents/vscode/CodeWiki/codewiki/templates/github_pages/index.html`
-
-## Conclusion
-
-These improvements bring the CLI's HTMLGenerator in line with the frontend implementation while maintaining backward compatibility and adding new features. The auto-loading functionality significantly simplifies usage, while the enhanced navigation and metadata display provide a better user experience.
-
diff --git a/README.md b/README.md
index 2cb662f4..cb60f1be 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# CodeWiki: Automated Repository-Level Documentation Generation
+# CodeWiki: Automated Repository-Level Documentation at Scale
 
 
      
 
@@ -10,7 +10,7 @@
 
 **The first open-source framework for holistic, structured repository-level documentation across multilingual codebases**
 
-[Features](#features) • [Installation](#installation) • [Quick Start](#quick-start) • [Benchmark](#benchmark) • [Documentation](#documentation-structure) • [Demo](https://fsoft-ai4code.github.io/codewiki-demo/) • [Citation](#citation)
+[Features](#features) • [Installation](#installation) • [Quick Start](#quick-start) • [Benchmark](#benchmark) • [Demo](https://fsoft-ai4code.github.io/codewiki-demo/) • [Citation](#citation)
 
 
      
 
@@ -40,14 +40,31 @@ Developers spend **58% of their working time** understanding codebases, yet main
 
 ## Features
 
-### Documentation Generation
+### 🎯 Two Usage Modes
+
+CodeWiki offers both **CLI** and **Web Application** interfaces:
+
+#### CLI Tool
+- ✅ **Local Projects** - Generate documentation for codebases on your machine
+- ✅ **Git Integration** - Automated branch creation and commit management
+- ✅ **Secure Configuration** - API keys stored in system keychain
+- ✅ **GitHub Pages** - Generate beautiful, interactive HTML documentation
+- ✅ **Progress Tracking** - Real-time progress updates with ETA
+
+#### Web Application
+- ✅ **GitHub URL Input** - Generate docs by repository URL and optional commit ID
+- ✅ **Remote Processing** - No need to clone repositories locally
+- ✅ **Web Interface** - Easy-to-use browser-based UI
+
+### 📚 Documentation Generation
 
 - ✅ **Repository-Level Documentation** - First framework to generate complete repo-level docs at scale
 - ✅ **Visual Artifacts** - Automatic generation of architecture diagrams and data flow visualizations
 - ✅ **Cross-Module References** - Intelligent reference management prevents redundancy
 - ✅ **Hierarchical Structure** - Multi-level documentation from high-level overviews to detailed APIs
+- ✅ **Mermaid Diagrams** - Automatic generation of architecture and data flow diagrams
 
-### Benchmark
+### 🔬 Benchmark
 
 - ✅ **[CodeWikiBench](https://github.com/FSoft-AI4Code/CodeWikiBench.git)** - First benchmark specifically designed for repository-level documentation
 
@@ -72,9 +89,26 @@ CodeWiki demonstrates significant improvements in high-level and managed languag
 
 - Python 3.12+
 - Node.js (for mermaid validation)
+- Git (optional, for CLI branch management)
 - Docker (optional, for containerized deployment)
 
-### Standard Installation
+### CLI Installation
+
+Install CodeWiki CLI from source:
+
+```bash
+pip install https://github.com/FSoft-AI4Code/CodeWiki.git
+```
+
+Verify installation:
+
+```bash
+codewiki --version
+```
+
+### Web Application Installation
+
+For the web interface:
 
 ```bash
 # Clone the repository
@@ -85,7 +119,7 @@ cd codewiki
 # macOS
 brew install node
 # Linux
-sudo apt update && apt install -y nodejs npm
+sudo apt update && sudo apt install -y nodejs npm
 
 # Create and activate virtual environment
 python3.12 -m venv .venv
@@ -94,27 +128,126 @@ source .venv/bin/activate  # On Windows: .venv\Scripts\activate
 # Install dependencies
 pip install -r requirements.txt
 
-#Create a `.env` file from the template and edit with your configuration
-cp env.example .env
+# Create a `.env` file from the template
+cp docker/env.example .env
+# Edit .env with your API keys and configuration
 
 # Start the web application
-python run_web_app.py
+python codewiki/run_web_app.py
 
-#Access the application at `http://localhost:8000` to generate documentation by github url and commit id (optional)
+# Access at http://localhost:8000
 ```
 
 ### Docker Installation
 
+For containerized deployment, see [Docker Setup](#docker-deployment) section below.
+
+---
+
+## Quick Start
+
+### CLI Usage
+
+#### 1. Configure CodeWiki
+
+```bash
+codewiki config set \
+  --api-key YOUR_API_KEY \
+  --base-url https://api.anthropic.com \
+  --main-model claude-sonnet-4 \
+  --cluster-model claude-sonnet-4
+```
+
+Verify configuration:
+
 ```bash
-# Copy environment configuration
-cp env.example .env
-# Edit .env with your API keys
+codewiki config show
+codewiki config validate
+```
+
+#### 2. Generate Documentation
 
-# Create network
-docker network create codewiki-network
+```bash
+cd /path/to/your/project
+codewiki generate
+```
 
-# Start services
-docker-compose up -d
+Documentation will be created in `./docs/`
+
+#### 3. Generate with GitHub Pages
+
+```bash
+codewiki generate --github-pages
+```
+
+This creates an interactive HTML viewer at `./docs/index.html`
+
+---
+
+## CLI Commands
+
+### Configuration Management
+
+```bash
+# Set configuration
+codewiki config set --api-key  --base-url  \
+  --main-model  --cluster-model 
+
+# Show configuration
+codewiki config show
+
+# Validate configuration
+codewiki config validate
+```
+
+### Documentation Generation
+
+```bash
+# Basic generation
+codewiki generate
+
+# Custom output directory
+codewiki generate --output ./documentation
+
+# Create git branch
+codewiki generate --create-branch
+
+# Generate GitHub Pages HTML
+codewiki generate --github-pages
+
+# Full-featured
+codewiki generate --create-branch --github-pages --verbose
+```
+
+---
+
+## Configuration
+
+### CLI Configuration
+
+Configuration is stored in:
+- API keys: System keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service)
+- Settings: `~/.codewiki/config.json`
+
+### Web Application Configuration
+
+Configuration uses environment variables in `.env` file:
+
+```bash
+# LLM API Configuration
+MAIN_MODEL=claude-sonnet-4
+FALLBACK_MODEL_1=glm-4p5
+CLUSTER_MODEL=claude-sonnet-4
+LLM_BASE_URL=http://litellm:4000/
+LLM_API_KEY=sk-1234
+
+# Application Port
+APP_PORT=8000
+
+# Optional: Logfire Configuration
+LOGFIRE_TOKEN=
+LOGFIRE_PROJECT_NAME=codewiki
+LOGFIRE_SERVICE_NAME=codewiki
 ```
 
 ---
@@ -164,6 +297,121 @@ Generated documentation includes:
 - **Sequence Diagrams** - Inter-component communication patterns
 - **Dependency Graphs** - Module and function dependencies
 
+### Output Structure
+
+```
+./docs/
+├── overview.md              # Start here!
+├── module1.md               # Module documentation
+├── module2.md
+├── ...
+├── module_tree.json         # Module hierarchy
+├── first_module_tree.json   # Initial clustering
+├── metadata.json            # Generation details
+└── index.html              # GitHub Pages viewer (if --github-pages)
+```
+
+---
+
+## Supported Languages
+
+| Language   | Extensions          | Support |
+|------------|---------------------|---------|
+| Python     | `.py`               | Full    |
+| Java       | `.java`             | Full    |
+| JavaScript | `.js`, `.jsx`       | Full    |
+| TypeScript | `.ts`, `.tsx`       | Full    |
+| C          | `.c`, `.h`          | Full    |
+| C++        | `.cpp`, `.hpp`, etc.| Full    |
+| C#         | `.cs`               | Full    |
+
+---
+
+## Docker Deployment
+
+### Quick Start with Docker
+
+1. **Set up environment variables**:
+   ```bash
+   # Copy from docker directory
+   cp docker/env.example .env
+   # Edit .env with your configuration
+   ```
+
+2. **Create Docker network**:
+   ```bash
+   docker network create codewiki-network
+   ```
+
+3. **Start the services**:
+   ```bash
+   # From project root
+   docker-compose -f docker/docker-compose.yml up -d
+   
+   # Or from docker directory
+   cd docker
+   docker-compose up -d
+   ```
+
+4. **Access the application**:
+   - Web app: http://localhost:8000
+
+### Docker Configuration
+
+All Docker-related files are in the `docker/` directory:
+- `docker/Dockerfile` - Container image definition
+- `docker/docker-compose.yml` - Service orchestration
+- `docker/env.example` - Environment variables template
+
+### Stopping Services
+
+```bash
+# From project root
+docker-compose -f docker/docker-compose.yml down
+
+# Or from docker directory
+cd docker
+docker-compose down
+```
+
+---
+
+## Development
+
+### Project Structure
+
+```
+codewiki/
+├── codewiki/                 # Main package
+│   ├── cli/                  # CLI implementation
+│   │   ├── commands/         # CLI commands (config, generate)
+│   │   ├── models/           # Data models
+│   │   ├── utils/            # Utilities
+│   │   └── adapters/         # External integrations
+│   ├── src/                  # Web application
+│   │   ├── be/               # Backend (dependency analysis, agents)
+│   │   └── fe/               # Frontend (web interface)
+│   ├── templates/            # HTML templates
+│   └── run_web_app.py        # Web app entry point
+├── docker/                   # Docker configuration
+│   ├── Dockerfile
+│   ├── docker-compose.yml
+│   └── env.example
+├── tests/                    # Test suite
+├── output/                   # Generated documentation output
+└── README.md                 # This file
+```
+
+---
+
+## Requirements
+
+- Python 3.12+
+- Git (optional, for branch management)
+- LLM API access (Anthropic Claude, OpenAI, etc.)
+- Tree-sitter language parsers (automatically installed)
+- System keychain support for CLI (macOS Keychain, Windows Credential Manager, Linux Secret Service)
+- Node.js (for mermaid diagram validation)
 
 ---
 
@@ -173,29 +421,25 @@ If you use CodeWiki in your research, please cite:
 
 ```bibtex
 @article{codewiki2025,
-  title={CodeWiki: Automated Repository-Level Documentation Generation with Hierarchical Decomposition and Agentic Processing},
+  title={CodeWiki: Automated Repository-Level Documentation at Scale},
   author={Your Name},
   journal={arXiv preprint arXiv:XXXXX},
   year={2025}
 }
 ```
 
-
+## License
 
+MIT License - see LICENSE file for details
 
-
+- **Live Demo**: [View documentation examples](https://fsoft-ai4code.github.io/codewiki-demo/)
+- **Issues**: https://github.com/yourusername/codewiki/issues
 
 ---
 
@@ -203,6 +447,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 **Made with ❤️ by the CodeWiki Team**
 
-[⬆ Back to Top](#codewiki-automated-repository-level-documentation-generation)
+[⬆ Back to Top](#codewiki-automated-repository-level-documentation-at-scale)
 
 
      
diff --git a/codewiki/README.md b/codewiki/README.md
deleted file mode 100644
index cbf2bc44..00000000
--- a/codewiki/README.md
+++ /dev/null
@@ -1,138 +0,0 @@
-# CodeWiki CLI
-
-Transform your codebase into comprehensive documentation using AI-powered analysis.
-
-## Features
-
-- **Multi-Language Support**: Python, Java, JavaScript, TypeScript, C, C++, C#
-- **AI-Powered Analysis**: Uses LLM models for intelligent documentation generation
-- **Comprehensive Documentation**: Generates overview, module docs, and architecture diagrams
-- **Dependency Analysis**: Tree-sitter based code parsing and dependency graph generation
-- **Module Clustering**: Intelligent grouping of related code components
-- **GitHub Pages Integration**: Generate beautiful, interactive HTML documentation
-- **Git Workflow Support**: Automated branch creation and commit management
-- **Secure Configuration**: API keys stored in system keychain
-- **Progress Tracking**: Real-time progress updates with ETA
-- **Mermaid Diagrams**: Automatic generation of architecture and data flow diagrams
-
-## Installation
-
-```bash
-pip install codewiki
-```
-
-## Quick Start
-
-### 1. Configure CodeWiki
-
-```bash
-codewiki config set \
-  --api-key YOUR_API_KEY \
-  --base-url https://api.anthropic.com \
-  --main-model claude-sonnet-4 \
-  --cluster-model claude-sonnet-4
-```
-
-### 2. Generate Documentation
-
-```bash
-cd /path/to/your/project
-codewiki generate
-```
-
-Documentation will be created in `./docs/`
-
-### 3. Generate with GitHub Pages
-
-```bash
-codewiki generate --github-pages
-```
-
-This creates an interactive HTML viewer at `./docs/index.html`
-
-## Commands
-
-### Configuration Management
-
-```bash
-# Set configuration
-codewiki config set --api-key  --base-url  \
-  --main-model  --cluster-model 
-
-# Show configuration
-codewiki config show
-
-# Validate configuration
-codewiki config validate
-```
-
-### Documentation Generation
-
-```bash
-# Basic generation
-codewiki generate
-
-# Custom output directory
-codewiki generate --output ./documentation
-
-# Create git branch
-codewiki generate --create-branch
-
-# Generate GitHub Pages HTML
-codewiki generate --github-pages
-
-# Full-featured
-codewiki generate --create-branch --github-pages --verbose
-```
-
-## Configuration
-
-Configuration is stored in:
-- API keys: System keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service)
-- Settings: `~/.codewiki/config.json`
-
-## Supported Languages
-
-| Language   | Extensions          |
-|------------|---------------------|
-| Python     | `.py`               |
-| Java       | `.java`             |
-| JavaScript | `.js`, `.jsx`       |
-| TypeScript | `.ts`, `.tsx`       |
-| C          | `.c`, `.h`          |
-| C++        | `.cpp`, `.hpp`, etc.|
-| C#         | `.cs`               |
-
-## Requirements
-
-- Python 3.12+
-- Git (optional, for branch management)
-- LLM API access (Anthropic Claude, OpenAI, etc.)
-- Tree-sitter language parsers (automatically installed)
-- System keychain support (macOS Keychain, Windows Credential Manager, Linux Secret Service)
-
-## Development
-
-### Install from Source
-
-```bash
-git clone https://github.com/yourusername/codewiki.git
-cd codewiki
-pip install -e .
-```
-
-### Run Tests
-
-```bash
-pytest tests/
-```
-
-## License
-
-MIT License - see LICENSE file for details
-
-## Support
-
-- Documentation: https://github.com/yourusername/codewiki/docs
-- Issues: https://github.com/yourusername/codewiki/issues
-
diff --git a/.dockerignore b/docker/.dockerignore
similarity index 100%
rename from .dockerignore
rename to docker/.dockerignore
diff --git a/docker/DOCKER_README.md b/docker/DOCKER_README.md
new file mode 100644
index 00000000..c45d7f06
--- /dev/null
+++ b/docker/DOCKER_README.md
@@ -0,0 +1,443 @@
+# CodeWiki Docker Setup
+
+This document explains how to run CodeWiki using Docker and Docker Compose.
+
+## Overview
+
+The Docker setup provides a containerized environment for running the CodeWiki web application, which allows you to generate documentation for GitHub repositories through a web interface.
+
+## File Structure
+
+All Docker-related files are located in the `docker/` directory:
+
+```
+docker/
+├── Dockerfile           # Container image definition
+├── docker-compose.yml   # Service orchestration
+├── env.example          # Environment variables template
+└── DOCKER_README.md     # This file
+```
+
+The Dockerfile builds from the project root context to include all necessary application code.
+
+---
+
+## Quick Start
+
+### 1. Clone the Repository
+
+```bash
+git clone 
+cd CodeWiki
+```
+
+### 2. Set Up Environment Variables
+
+```bash
+# Copy the example environment file
+cp docker/env.example .env
+
+# Edit .env file with your configuration
+nano .env  # or use your preferred editor
+```
+
+Required configuration in `.env`:
+
+```bash
+# LLM API Configuration
+MAIN_MODEL=claude-sonnet-4
+FALLBACK_MODEL_1=glm-4p5
+CLUSTER_MODEL=claude-sonnet-4
+LLM_BASE_URL=https://api.anthropic.com  # or your LiteLLM proxy
+LLM_API_KEY=your-api-key-here
+
+# Application Port
+APP_PORT=8000
+
+# Optional: Logfire Configuration (for monitoring)
+LOGFIRE_TOKEN=
+LOGFIRE_PROJECT_NAME=codewiki
+LOGFIRE_SERVICE_NAME=codewiki
+```
+
+### 3. Create Docker Network
+
+```bash
+docker network create codewiki-network
+```
+
+### 4. Start the Services
+
+**Option A: From project root**
+```bash
+docker-compose -f docker/docker-compose.yml up -d
+```
+
+**Option B: From docker directory**
+```bash
+cd docker
+docker-compose up -d
+```
+
+### 5. Access the Application
+
+- Web Application: http://localhost:8000
+
+The application will be available at the port specified in your `.env` file (default: 8000).
+
+---
+
+## Docker Compose Configuration
+
+The `docker-compose.yml` file defines the CodeWiki service with the following features:
+
+### Service Configuration
+
+- **Image**: `codewiki:0.0.1`
+- **Build Context**: Parent directory (`.` relative to docker/)
+- **Container Name**: `codewiki`
+- **Port Mapping**: `${APP_PORT:-8000}:8000`
+- **Network**: `codewiki-network` (external)
+
+### Environment Variables
+
+The container uses environment variables from the `.env` file:
+- `PYTHONPATH=/app/src` - Set Python module path
+- `PYTHONUNBUFFERED=1` - Enable real-time logging
+- All variables from `.env` file
+
+### Volume Mounts
+
+The following directories are mounted as volumes:
+
+```yaml
+volumes:
+  - ./output:/app/output        # Persistent storage for generated docs
+  - ~/.ssh:/root/.ssh:ro        # SSH keys for private repos (read-only)
+```
+
+**Note**: Git credentials can be mounted if needed for private repositories:
+```yaml
+  # Uncomment in docker-compose.yml if needed
+  - ~/.gitconfig:/root/.gitconfig:ro
+```
+
+### Health Check
+
+The service includes a health check that:
+- Runs every 30 seconds
+- Times out after 10 seconds
+- Retries 3 times on failure
+- Starts checking after 20 seconds
+
+### Restart Policy
+
+The container is set to restart automatically unless explicitly stopped (`restart: unless-stopped`).
+
+---
+
+## Dockerfile Details
+
+The Dockerfile (`docker/Dockerfile`) builds the CodeWiki image with:
+
+### Base Image
+- Python 3.12 slim image for smaller size
+
+### System Dependencies
+- `git` - For repository cloning
+- `curl` - For health checks
+- `nodejs` and `npm` - For mermaid diagram validation
+
+### Application Setup
+1. Copies `requirements.txt` first (for better caching)
+2. Installs Python dependencies
+3. Copies entire application code
+4. Creates output directories:
+   - `output/cache`
+   - `output/temp`
+   - `output/docs`
+   - `output/dependency_graphs`
+
+### Runtime Configuration
+- **Working Directory**: `/app`
+- **Exposed Port**: `8000`
+- **Entry Point**: `python codewiki/run_web_app.py --host 0.0.0.0 --port 8000`
+
+---
+
+## Common Operations
+
+### View Logs
+
+```bash
+# From project root
+docker-compose -f docker/docker-compose.yml logs -f
+
+# From docker directory
+cd docker
+docker-compose logs -f
+
+# View specific service
+docker logs codewiki -f
+```
+
+### Stop Services
+
+```bash
+# From project root
+docker-compose -f docker/docker-compose.yml stop
+
+# From docker directory
+cd docker
+docker-compose stop
+```
+
+### Stop and Remove Containers
+
+```bash
+# From project root
+docker-compose -f docker/docker-compose.yml down
+
+# From docker directory
+cd docker
+docker-compose down
+
+# Remove volumes as well
+docker-compose down -v
+```
+
+### Rebuild Image
+
+If you've made changes to the code or Dockerfile:
+
+```bash
+# From project root
+docker-compose -f docker/docker-compose.yml build --no-cache
+
+# From docker directory
+cd docker
+docker-compose build --no-cache
+
+# Rebuild and restart
+docker-compose up -d --build
+```
+
+### Access Container Shell
+
+```bash
+docker exec -it codewiki /bin/bash
+```
+
+---
+
+## Persistent Storage
+
+### Output Directory
+
+The `output/` directory is mounted as a volume, ensuring generated documentation persists across container restarts:
+
+```
+output/
+├── cache/                    # Cached dependency graphs and jobs
+├── docs/                     # Generated documentation
+├── dependency_graphs/        # JSON dependency graphs
+└── temp/                     # Temporary files
+```
+
+### SSH Keys
+
+If you need to clone private repositories, ensure your SSH keys are available:
+
+```bash
+# Verify SSH keys are accessible
+ls -la ~/.ssh/
+
+# The docker-compose.yml mounts ~/.ssh as read-only
+```
+
+---
+
+## Troubleshooting
+
+### Port Already in Use
+
+If port 8000 is already in use:
+
+```bash
+# Change APP_PORT in .env file
+echo "APP_PORT=8001" >> .env
+
+# Restart services
+docker-compose -f docker/docker-compose.yml down
+docker-compose -f docker/docker-compose.yml up -d
+```
+
+### Container Won't Start
+
+Check logs for errors:
+
+```bash
+docker logs codewiki
+```
+
+Common issues:
+- **Invalid API key**: Verify `LLM_API_KEY` in `.env`
+- **Network not found**: Create network with `docker network create codewiki-network`
+- **Port conflict**: Change `APP_PORT` in `.env`
+
+### Health Check Failing
+
+```bash
+# Check if the application is responding
+curl http://localhost:8000/
+
+# Check container health status
+docker inspect codewiki --format='{{.State.Health.Status}}'
+
+# View health check logs
+docker inspect codewiki --format='{{range .State.Health.Log}}{{.Output}}{{end}}'
+```
+
+### Permission Issues with Volumes
+
+If you encounter permission issues with mounted volumes:
+
+```bash
+# On Linux, ensure proper ownership
+sudo chown -R $(id -u):$(id -g) output/
+
+# Or run container with user mapping
+docker-compose -f docker/docker-compose.yml down
+# Add to docker-compose.yml under 'codewiki' service:
+# user: "${UID}:${GID}"
+```
+
+### Private Repository Access
+
+For private repositories:
+
+1. Ensure SSH keys are properly mounted:
+   ```yaml
+   volumes:
+     - ~/.ssh:/root/.ssh:ro
+   ```
+
+2. Verify key permissions:
+   ```bash
+   chmod 600 ~/.ssh/id_rsa
+   chmod 644 ~/.ssh/id_rsa.pub
+   ```
+
+3. Add GitHub to known_hosts:
+   ```bash
+   docker exec -it codewiki ssh-keyscan github.com >> /root/.ssh/known_hosts
+   ```
+
+---
+
+## Production Deployment
+
+### Security Considerations
+
+1. **Environment Variables**: Never commit `.env` file to version control
+2. **API Keys**: Use secrets management in production
+3. **Network**: Use isolated Docker networks
+4. **Volumes**: Set appropriate permissions on mounted volumes
+5. **Updates**: Regularly update base image and dependencies
+
+### Recommended Production Setup
+
+```yaml
+# Use secrets instead of .env file
+services:
+  codewiki:
+    secrets:
+      - llm_api_key
+    environment:
+      - LLM_API_KEY_FILE=/run/secrets/llm_api_key
+
+secrets:
+  llm_api_key:
+    external: true
+```
+
+### Resource Limits
+
+Add resource limits in production:
+
+```yaml
+services:
+  codewiki:
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: 4G
+        reservations:
+          cpus: '1'
+          memory: 2G
+```
+
+### Reverse Proxy
+
+Use nginx or traefik as a reverse proxy:
+
+```yaml
+services:
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./certs:/etc/nginx/certs:ro
+```
+
+---
+
+## Integration with External Services
+
+### Using LiteLLM Proxy
+
+If using a LiteLLM proxy for LLM API management:
+
+```bash
+# In .env file
+LLM_BASE_URL=http://litellm:4000/
+LLM_API_KEY=sk-your-proxy-key
+
+# Add LiteLLM service to docker-compose.yml
+services:
+  litellm:
+    image: ghcr.io/berriai/litellm:latest
+    ports:
+      - "4000:4000"
+    environment:
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+    networks:
+      - codewiki-network
+```
+
+---
+
+## More Information
+
+- **Main Documentation**: See [../README.md](../README.md) for complete feature list and usage
+- **CLI Tool**: For command-line documentation generation
+- **Web Interface**: For GitHub URL-based documentation generation
+
+---
+
+## Support
+
+For issues related to Docker deployment:
+1. Check logs: `docker logs codewiki`
+2. Verify configuration: `docker exec codewiki env | grep -E '(LLM|APP)'`
+3. Test connectivity: `docker exec codewiki curl -I http://localhost:8000`
+4. Report issues: https://github.com/yourusername/codewiki/issues
+
+---
+
+**Happy documenting with Docker! 🐳📚**
diff --git a/Dockerfile b/docker/Dockerfile
similarity index 77%
rename from Dockerfile
rename to docker/Dockerfile
index 04ee4de2..d0a70d16 100644
--- a/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-# Use Python 3.11 slim image as base
+# Use Python 3.12 slim image as base
 FROM python:3.12-slim
 
 # Set working directory
@@ -19,13 +19,16 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy application code
-COPY . .
+COPY codewiki ./codewiki
+COPY img ./img
+COPY pyproject.toml .
+COPY README.md .
 
 # Create output directories
 RUN mkdir -p output/cache output/temp output/docs output/dependency_graphs
 
 # Set environment variables
-ENV PYTHONPATH=/app/src
+ENV PYTHONPATH=/app
 ENV PYTHONUNBUFFERED=1
 
 # Expose port
@@ -36,4 +39,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
     CMD curl -f http://localhost:8000/ || exit 1
 
 # Default command
-CMD ["python", "run_web_app.py", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "codewiki/run_web_app.py", "--host", "0.0.0.0", "--port", "8000"]
diff --git a/docker-compose.yml b/docker/docker-compose.yml
similarity index 84%
rename from docker-compose.yml
rename to docker/docker-compose.yml
index 33091789..28084e7f 100644
--- a/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -2,21 +2,21 @@ services:
   codewiki:
     image: codewiki:0.0.1
     build:
-      context: .
-      dockerfile: Dockerfile
+      context: ..
+      dockerfile: docker/Dockerfile
     container_name: codewiki
     ports:
       - "${APP_PORT:-8000}:8000"
     environment:
-      - PYTHONPATH=/app/src
+      - PYTHONPATH=/app
       - PYTHONUNBUFFERED=1
     env_file:
-      - .env
+      - ../.env
     networks:
       - net
     volumes:
       # Persistent storage for cache and output
-      - ./output:/app/output
+      - ../output:/app/output
       # Git credentials (if needed for private repos)
       # - ~/.gitconfig:/root/.gitconfig:ro
       - ~/.ssh:/root/.ssh:ro
diff --git a/env.example b/docker/env.example
similarity index 100%
rename from env.example
rename to docker/env.example