Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions src/apify_client/_resource_clients/_resource_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from datetime import UTC, datetime, timedelta
from functools import cached_property
from typing import TYPE_CHECKING, Any, Literal, get_args
from urllib.parse import urlencode, urlparse, urlunparse

from apify_client._consts import DEFAULT_WAIT_FOR_FINISH, DEFAULT_WAIT_WHEN_JOB_NOT_EXIST
from apify_client._docs import docs_group
Expand Down Expand Up @@ -115,6 +116,25 @@ def _build_url(

return url

def _build_public_url(self, path: str, params: dict[str, Any]) -> str:
"""Build a public resource URL with API-normalized query params.

Normalizes `params` the same way the HTTP request path does (bool -> true/false, list -> comma-joined,
datetime -> ISO 8601 Zulu) so the shareable URL matches what the client would send over the wire.

Args:
path: Path segment appended to the resource URL (e.g. 'items', 'keys').
params: Query parameters to normalize and append.

Returns:
The public URL with a normalized query string.
"""
public_url = urlparse(self._build_url(path, public=True))
filtered_params = self._http_client._parse_params(params) # noqa: SLF001
Comment thread
Pijukatel marked this conversation as resolved.
if filtered_params:
public_url = public_url._replace(query=urlencode(filtered_params))
return urlunparse(public_url)

def _build_params(self, **kwargs: Any) -> dict:
"""Merge default params with method params, filtering out None values.

Expand All @@ -133,7 +153,7 @@ def _clean_json_payload(data: dict) -> dict:

The Apify API ignores missing fields but may reject fields explicitly set to None.
Nested sub-models serialized by Pydantic may produce empty dicts when all their
fields are None these are also removed.
fields are None - these are also removed.

Uses an iterative stack-based approach, analogous to _build_params for query params.
"""
Expand Down Expand Up @@ -203,7 +223,7 @@ def _get(self, *, timeout: Timeout) -> dict | None:
"""Perform a GET request for this resource, returning the parsed response or None if not found.

404s collapse to `None` only for ID-identified clients. Chained clients without a `resource_id`
(e.g. `run.dataset()`) propagate `NotFoundError` see `catch_not_found_for_resource_or_throw`.
(e.g. `run.dataset()`) propagate `NotFoundError` - see `catch_not_found_for_resource_or_throw`.
"""
try:
response = self._http_client.call(
Expand Down Expand Up @@ -232,7 +252,7 @@ def _delete(self, *, timeout: Timeout) -> None:
"""Perform a DELETE request to delete this resource.

404s are swallowed (idempotent DELETE) only for ID-identified clients. Chained clients without a
`resource_id` propagate `NotFoundError` see `catch_not_found_for_resource_or_throw`.
`resource_id` propagate `NotFoundError` - see `catch_not_found_for_resource_or_throw`.
"""
try:
self._http_client.call(
Expand Down Expand Up @@ -395,7 +415,7 @@ async def _get(self, *, timeout: Timeout) -> dict | None:
"""Perform a GET request for this resource, returning the parsed response or None if not found.

404s collapse to `None` only for ID-identified clients. Chained clients without a `resource_id`
(e.g. `run.dataset()`) propagate `NotFoundError` see `catch_not_found_for_resource_or_throw`.
(e.g. `run.dataset()`) propagate `NotFoundError` - see `catch_not_found_for_resource_or_throw`.
"""
try:
response = await self._http_client.call(
Expand Down Expand Up @@ -424,7 +444,7 @@ async def _delete(self, *, timeout: Timeout) -> None:
"""Perform a DELETE request to delete this resource.

404s are swallowed (idempotent DELETE) only for ID-identified clients. Chained clients without a
`resource_id` propagate `NotFoundError` see `catch_not_found_for_resource_or_throw`.
`resource_id` propagate `NotFoundError` - see `catch_not_found_for_resource_or_throw`.
"""
try:
await self._http_client.call(
Expand Down
15 changes: 2 additions & 13 deletions src/apify_client/_resource_clients/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlparse, urlunparse

from apify_client._docs import docs_group
from apify_client._models import Dataset, DatasetResponse, DatasetStatistics, DatasetStatisticsResponse
Expand Down Expand Up @@ -606,12 +605,7 @@ def create_items_public_url(
)
request_params['signature'] = signature

items_public_url = urlparse(self._build_url('items', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}
if filtered_params:
items_public_url = items_public_url._replace(query=urlencode(filtered_params))

return urlunparse(items_public_url)
return self._build_public_url('items', request_params)


@docs_group('Resource clients')
Expand Down Expand Up @@ -1172,9 +1166,4 @@ async def create_items_public_url(
)
request_params['signature'] = signature

items_public_url = urlparse(self._build_url('items', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}
if filtered_params:
items_public_url = items_public_url._replace(query=urlencode(filtered_params))

return urlunparse(items_public_url)
return self._build_public_url('items', request_params)
33 changes: 4 additions & 29 deletions src/apify_client/_resource_clients/key_value_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from contextlib import asynccontextmanager, contextmanager
from http import HTTPStatus
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlparse, urlunparse

from apify_client._docs import docs_group
from apify_client._models import (
Expand Down Expand Up @@ -425,13 +424,7 @@ def get_record_public_url(self, key: str, *, timeout: Timeout = 'long') -> str:
if metadata and metadata.url_signing_secret_key:
request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)

key_public_url = urlparse(self._build_url(f'records/{key}', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}

if filtered_params:
key_public_url = key_public_url._replace(query=urlencode(filtered_params))

return urlunparse(key_public_url)
return self._build_public_url(f'records/{key}', request_params)

def create_keys_public_url(
self,
Expand Down Expand Up @@ -482,13 +475,7 @@ def create_keys_public_url(
)
request_params['signature'] = signature

keys_public_url = urlparse(self._build_url('keys', public=True))

filtered_params = {k: v for k, v in request_params.items() if v is not None}
if filtered_params:
keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))

return urlunparse(keys_public_url)
return self._build_public_url('keys', request_params)


@docs_group('Resource clients')
Expand Down Expand Up @@ -853,13 +840,7 @@ async def get_record_public_url(self, key: str, *, timeout: Timeout = 'long') ->
if metadata and metadata.url_signing_secret_key:
request_params['signature'] = create_hmac_signature(metadata.url_signing_secret_key, key)

key_public_url = urlparse(self._build_url(f'records/{key}', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}

if filtered_params:
key_public_url = key_public_url._replace(query=urlencode(filtered_params))

return urlunparse(key_public_url)
return self._build_public_url(f'records/{key}', request_params)

async def create_keys_public_url(
self,
Expand Down Expand Up @@ -910,10 +891,4 @@ async def create_keys_public_url(
)
request_params['signature'] = signature

keys_public_url = urlparse(self._build_url('keys', public=True))

filtered_params = {k: v for k, v in request_params.items() if v is not None}
if filtered_params:
keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))

return urlunparse(keys_public_url)
return self._build_public_url('keys', request_params)
35 changes: 35 additions & 0 deletions tests/unit/test_url_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
from unittest import mock
from unittest.mock import Mock
from urllib.parse import parse_qs, urlparse

import pytest

Expand Down Expand Up @@ -128,6 +129,40 @@ async def test_dataset_public_url_async(api_url: str, api_public_url: str | None
)


def test_dataset_public_url_normalizes_params_sync() -> None:
"""Bool and list query params must be API-normalized (bool→true/false, list→comma-joined), not Python reprs."""
client = ApifyClient(token='dummy-token', api_url='https://api.apify.com')
dataset = client.dataset('someID')

mock_response = Mock()
mock_response.json.return_value = json.loads(MOCKED_DATASET_RESPONSE)

with mock.patch.object(client._http_client, 'call', return_value=mock_response):
public_url = dataset.create_items_public_url(clean=True, desc=False, fields=['title', 'url'])

query = parse_qs(urlparse(public_url).query)
assert query['clean'] == ['true']
assert query['desc'] == ['false']
assert query['fields'] == ['title,url']


async def test_dataset_public_url_normalizes_params_async() -> None:
"""Bool and list query params must be API-normalized (bool→true/false, list→comma-joined), not Python reprs."""
client = ApifyClientAsync(token='dummy-token', api_url='https://api.apify.com')
dataset = client.dataset('someID')

mock_response = Mock()
mock_response.json.return_value = json.loads(MOCKED_DATASET_RESPONSE)

with mock.patch.object(client._http_client, 'call', return_value=mock_response):
public_url = await dataset.create_items_public_url(clean=True, desc=False, fields=['title', 'url'])

query = parse_qs(urlparse(public_url).query)
assert query['clean'] == ['true']
assert query['desc'] == ['false']
assert query['fields'] == ['title,url']


# ============================================================================
# Key-value store URL generation tests
# ============================================================================
Expand Down
Loading