Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,7 @@ cli = [
"cookiecutter>=2.6.0",
"inquirer>=3.3.0",
"rich>=13.9.0",
# Pinned below 0.26, which breaks our `click.Choice` CLI options.
# See https://github.com/apify/crawlee-python/issues/2063.
"typer>=0.12.0,<0.26",
"typer>=0.26.0",
]
curl-impersonate = ["curl-cffi>=0.9.0"]
httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"]
Expand Down
114 changes: 66 additions & 48 deletions src/crawlee/_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@
import json
import sys
from pathlib import Path
from typing import Annotated, cast

from click import Choice
from typing import TYPE_CHECKING, Annotated, Literal, cast

try:
import inquirer
Expand All @@ -34,6 +32,15 @@
default_enable_apify_integration = cookiecutter_json['enable_apify_integration']
default_install_project = cookiecutter_json['install_project']

if TYPE_CHECKING:
# A `Literal` built from runtime data can't be introspected by PEP 586,
# so during type checking these aliases are plain `str`.
CrawlerType = HttpClientType = PackageManagerType = str
else:
CrawlerType = Literal[tuple(crawler_choices)]
HttpClientType = Literal[tuple(http_client_choices)]
PackageManagerType = Literal[tuple(package_manager_choices)]
Comment on lines +39 to +42


@cli.callback(invoke_without_command=True)
def callback(
Expand Down Expand Up @@ -124,52 +131,63 @@ def _prompt_bool(message: str, *, default: bool) -> bool:

@cli.command()
def create(
project_name: str | None = typer.Argument(
default=None,
show_default=False,
help='The name of the project and the directory that will be created to contain it. '
'If none is given, you will be prompted.',
),
crawler_type: str | None = typer.Option(
None,
'--crawler-type',
'--template',
show_default=False,
click_type=Choice(crawler_choices),
help='The library that will be used for crawling in your crawler. If none is given, you will be prompted.',
),
http_client: str | None = typer.Option(
None,
show_default=False,
click_type=Choice(http_client_choices),
help='The library that will be used to make HTTP requests in your crawler. '
'If none is given, you will be prompted.',
),
package_manager: str | None = typer.Option(
default=None,
show_default=False,
click_type=Choice(package_manager_choices),
help='Package manager to be used in the new project. If none is given, you will be prompted.',
),
start_url: str | None = typer.Option(
default=None,
show_default=False,
metavar='[START_URL]',
help='The URL where crawling should start. If none is given, you will be prompted.',
),
project_name: Annotated[
str | None,
typer.Argument(
show_default=False,
help='The name of the project and the directory that will be created to contain it. '
'If none is given, you will be prompted.',
),
] = None,
crawler_type: Annotated[
CrawlerType | None,
typer.Option(
'--crawler-type',
'--template',
show_default=False,
help='The library that will be used for crawling in your crawler. If none is given, you will be prompted.',
),
] = None,
http_client: Annotated[
HttpClientType | None,
typer.Option(
show_default=False,
help='The library that will be used to make HTTP requests in your crawler. '
'If none is given, you will be prompted.',
),
] = None,
package_manager: Annotated[
PackageManagerType | None,
typer.Option(
show_default=False,
help='Package manager to be used in the new project. If none is given, you will be prompted.',
),
] = None,
start_url: Annotated[
str | None,
typer.Option(
show_default=False,
metavar='[START_URL]',
help='The URL where crawling should start. If none is given, you will be prompted.',
),
] = None,
*,
enable_apify_integration: bool | None = typer.Option(
None,
'--apify/--no-apify',
show_default=False,
help='Should Apify integration be set up for you? If not given, you will be prompted.',
),
install_project: bool | None = typer.Option(
None,
'--install/--no-install',
show_default=False,
help='Should the project be installed now? If not given, you will be prompted.',
),
enable_apify_integration: Annotated[
bool | None,
typer.Option(
'--apify/--no-apify',
show_default=False,
help='Should Apify integration be set up for you? If not given, you will be prompted.',
),
] = None,
install_project: Annotated[
bool | None,
typer.Option(
'--install/--no-install',
show_default=False,
help='Should the project be installed now? If not given, you will be prompted.',
),
] = None,
) -> None:
"""Bootstrap a new Crawlee project."""
try:
Expand Down
10 changes: 5 additions & 5 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading