Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,12 @@ async def main() -> None:
soup = BeautifulSoup(response.content, 'html.parser')

# Push the extracted data to the default dataset.
await Actor.push_data({
'url': request.url,
'title': soup.title.string if soup.title else None,
})
await Actor.push_data(
{
'url': request.url,
'title': soup.title.string if soup.title else None,
}
)

# Mark the request as handled so it is not processed again.
await request_queue.mark_request_as_handled(request)
Expand All @@ -183,10 +185,12 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: PlaywrightCrawlingContext) -> None:
Actor.log.info(f'Scraping {context.request.url} ...')
await context.push_data({
'url': context.request.url,
'title': await context.page.title(),
})
await context.push_data(
{
'url': context.request.url,
'title': await context.page.title(),
}
)
# Follow links found on the page.
await context.enqueue_links()

Expand Down
13 changes: 9 additions & 4 deletions docs/04_upgrading/upgrading_to_v3.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ from crawlee.configuration import Configuration
from crawlee.events import LocalEventManager
from apify import Actor


async def main():

async with Actor():
Expand Down Expand Up @@ -125,15 +126,19 @@ Apify SDK v3.0 also reworks how storage clients are configured, giving you expli

```python
from crawlee import service_locator
from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient, MemoryStorageClient
from apify.storage_clients import (
ApifyStorageClient,
SmartApifyStorageClient,
MemoryStorageClient,
)
from apify import Actor


async def main():
service_locator.set_storage_client(
SmartApifyStorageClient(
cloud_storage_client=ApifyStorageClient(request_queue_access="single"),
local_storage_client=MemoryStorageClient()
cloud_storage_client=ApifyStorageClient(request_queue_access='single'),
local_storage_client=MemoryStorageClient(),
)
)
async with Actor:
Expand All @@ -158,7 +163,7 @@ async def main():
# Full client that supports multiple consumers of the Apify Request Queue
service_locator.set_storage_client(
SmartApifyStorageClient(
cloud_storage_client=ApifyStorageClient(request_queue_access="shared"),
cloud_storage_client=ApifyStorageClient(request_queue_access='shared'),
)
)
async with Actor:
Expand Down
8 changes: 6 additions & 2 deletions docs/04_upgrading/upgrading_to_v4.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,11 +227,15 @@ Secondary parameters on several client methods can no longer be passed positiona

```python
# Before (v3)
await client.key_value_store('my-store').set_record('my-key', {'data': 1}, 'application/json')
await client.key_value_store('my-store').set_record(
'my-key', {'data': 1}, 'application/json'
)
await client.run('my-run').charge('my-event', 5)

# After (v4)
await client.key_value_store('my-store').set_record('my-key', {'data': 1}, content_type='application/json')
await client.key_value_store('my-store').set_record(
'my-key', {'data': 1}, content_type='application/json'
)
await client.run('my-run').charge('my-event', count=5)
```

Expand Down
9 changes: 8 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,17 @@ allow-direct-references = true

[tool.ruff]
line-length = 120
include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]
include = [
"**/*.py",
# Ruff formats Python code blocks embedded in Markdown files.
"**/*.md",
"**/*.mdx",
]
exclude = [
"website/versioned_docs/**",
]
# MDX is Markdown; without this mapping Ruff would try to parse `.mdx` files as Python.
extension = { mdx = "markdown" }

[tool.ruff.lint]
select = ["ALL"]
Expand Down