From 3ff12812842b91072152169327aac28572c36ff3 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 28 Jul 2026 18:31:49 +0000 Subject: [PATCH] add cookies management guide --- .../cookie_management/initial_cookies.py | 30 +++++ .../cookie_management/persist_cookies.py | 41 +++++++ .../cookie_management/playwright_cookies.py | 40 +++++++ .../cookie_management/read_write_cookies.py | 36 ++++++ .../cookie_management/retry_pinned_session.py | 41 +++++++ .../retry_restore_cookies.py | 40 +++++++ .../cookie_management/retry_single_session.py | 41 +++++++ docs/guides/cookie_management.mdx | 106 ++++++++++++++++++ docs/guides/crawler_login.mdx | 2 +- docs/guides/http_headers.mdx | 2 +- docs/guides/session_management.mdx | 2 +- 11 files changed, 378 insertions(+), 3 deletions(-) create mode 100644 docs/guides/code_examples/cookie_management/initial_cookies.py create mode 100644 docs/guides/code_examples/cookie_management/persist_cookies.py create mode 100644 docs/guides/code_examples/cookie_management/playwright_cookies.py create mode 100644 docs/guides/code_examples/cookie_management/read_write_cookies.py create mode 100644 docs/guides/code_examples/cookie_management/retry_pinned_session.py create mode 100644 docs/guides/code_examples/cookie_management/retry_restore_cookies.py create mode 100644 docs/guides/code_examples/cookie_management/retry_single_session.py create mode 100644 docs/guides/cookie_management.mdx diff --git a/docs/guides/code_examples/cookie_management/initial_cookies.py b/docs/guides/code_examples/cookie_management/initial_cookies.py new file mode 100644 index 0000000000..83e8dafd0c --- /dev/null +++ b/docs/guides/code_examples/cookie_management/initial_cookies.py @@ -0,0 +1,30 @@ +import asyncio + +from crawlee.crawlers import HttpCrawler, HttpCrawlingContext +from crawlee.sessions import SessionPool + + +async def main() -> None: + crawler = HttpCrawler( + session_pool=SessionPool( + # Seed every new session in the pool with these cookies. + create_session_settings={ + 'cookies': [ + {'name': 'consent', 'value': 'accepted', 'domain': 'httpbingo.org'}, + {'name': 'locale', 'value': 'en_US', 'domain': 'httpbingo.org'}, + ], + }, + ), + ) + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + # The endpoint echoes back the cookies it received. + response = await context.http_response.read() + context.log.info(response.decode()) + + await crawler.run(['https://httpbingo.org/cookies']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/persist_cookies.py b/docs/guides/code_examples/cookie_management/persist_cookies.py new file mode 100644 index 0000000000..a6f1a23997 --- /dev/null +++ b/docs/guides/code_examples/cookie_management/persist_cookies.py @@ -0,0 +1,41 @@ +import asyncio +from datetime import timedelta + +from crawlee.crawlers import HttpCrawler, HttpCrawlingContext +from crawlee.sessions import SessionPool + + +async def main() -> None: + # Define the pool outside the crawler so its state can be read before the run. + # `persist_state_kvs_name` is required: an unnamed store is purged on start, + # so persistence would silently not survive a restart. + session_pool = SessionPool( + max_pool_size=1, + create_session_settings={ + # Keep the single session alive across runs. + 'max_usage_count': 999_999, + 'max_age': timedelta(hours=999_999), + 'max_error_score': 100, + }, + persistence_enabled=True, + persist_state_kvs_name='my-cookie-store', + ) + + async with session_pool: + # Read a session before crawling. On the first run its jar is empty. On + # later runs the cookies from the previous run are already restored. + session = await session_pool.get_session() + print(f'Cookies before run: {session.cookies.get_cookies_as_dicts()}') + + # With a single session, don't burn retries trying to rotate to another one. + crawler = HttpCrawler(max_session_rotations=0, session_pool=session_pool) + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + await crawler.run(['https://httpbingo.org/cookies/set?visited=1']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/playwright_cookies.py b/docs/guides/code_examples/cookie_management/playwright_cookies.py new file mode 100644 index 0000000000..115d519c47 --- /dev/null +++ b/docs/guides/code_examples/cookie_management/playwright_cookies.py @@ -0,0 +1,40 @@ +import asyncio + +from crawlee import Request +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext +from crawlee.sessions import SessionPool + + +async def main() -> None: + crawler = PlaywrightCrawler( + # A single session, so both requests run on the same one. + session_pool=SessionPool(max_pool_size=1), + ) + + @crawler.router.default_handler + async def handler(context: PlaywrightCrawlingContext) -> None: + if context.session is None: + return + + # The browser receives the `visited` cookie during the first navigation. + # It's synced back onto the session after the handler returns, so on the + # second request it's already on `context.session`. + names = [cookie['name'] for cookie in context.session.cookies] + context.log.info(f'Session cookies on {context.request.url}: {names}') + + if not context.request.user_data.get('followup'): + await context.add_requests( + [ + Request.from_url( + 'https://httpbingo.org/cookies', + user_data={'followup': True}, + always_enqueue=True, + ) + ] + ) + + await crawler.run(['https://httpbingo.org/cookies/set?visited=1']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/read_write_cookies.py b/docs/guides/code_examples/cookie_management/read_write_cookies.py new file mode 100644 index 0000000000..6f17840d22 --- /dev/null +++ b/docs/guides/code_examples/cookie_management/read_write_cookies.py @@ -0,0 +1,36 @@ +import asyncio + +from crawlee.crawlers import HttpCrawler, HttpCrawlingContext + + +async def main() -> None: + crawler = HttpCrawler() + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + if context.session is None: + return + + # Set a cookie on the session. It's sent with every following + # request that runs on this session. + context.session.cookies.set( + name='csrf_token', + value='abc123', + domain='httpbingo.org', + ) + + # Iterate over the cookies stored on the session, including ones the + # server set via `Set-Cookie` on the current or an earlier response. + for cookie in context.session.cookies: + context.log.info(f'{cookie["name"]}={cookie["value"]}') + + # Read a single cookie value by name. + token = context.session.cookies['csrf_token'] + context.log.info(f'CSRF token: {token}') + + # The server sets a `session_id` cookie and echoes the request cookies back. + await crawler.run(['https://httpbingo.org/cookies/set?session_id=42']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/retry_pinned_session.py b/docs/guides/code_examples/cookie_management/retry_pinned_session.py new file mode 100644 index 0000000000..2b29ada7be --- /dev/null +++ b/docs/guides/code_examples/cookie_management/retry_pinned_session.py @@ -0,0 +1,41 @@ +import asyncio + +from crawlee import Request +from crawlee.crawlers import HttpCrawler, HttpCrawlingContext + + +async def main() -> None: + crawler = HttpCrawler() + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + if context.session is None: + return + + if context.request.user_data.get('prepared'): + # Cookies from the setup step are on this pinned session. + context.log.info(f'Cookies: {context.session.cookies.get_cookies_as_dicts()}') + return + + # First pass: establish cookies on the current session. + await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') + + await context.add_requests( + [ + Request.from_url( + context.request.url, + # Bind the new request to the session that now has the cookies. + session_id=context.session.id, + # Mark the request so the handler skips the setup step on it. + user_data={'prepared': True}, + # Run the same URL again despite deduplication. + always_enqueue=True, + ) + ] + ) + + await crawler.run(['https://httpbingo.org/cookies']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py new file mode 100644 index 0000000000..6f785ee662 --- /dev/null +++ b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py @@ -0,0 +1,40 @@ +import asyncio +from typing import TYPE_CHECKING, cast + +from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext + +if TYPE_CHECKING: + from crawlee.sessions import CookieParam + + +async def main() -> None: + crawler = HttpCrawler() + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + if context.session is None: + return + + state = await context.use_state(default_value={}) + if not state.get('cookies'): + # First pass: establish cookies once and store them for every session, + # then raise to trigger a retry. + await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') + state['cookies'] = context.session.cookies.get_cookies_as_dicts() + raise RuntimeError('retry with cookies') + + context.log.info(f'Cookies: {context.session.cookies.get_cookies_as_dicts()}') + + @crawler.pre_navigation_hook + async def apply_cookies(context: BasicCrawlingContext) -> None: + # Runs before navigation. Apply the shared cookies to whichever session + # handles the request, so every session ends up with them. + state = await context.use_state(default_value={}) + if context.session and (cookies := state.get('cookies')): + context.session.cookies.set_cookies(cast('list[CookieParam]', cookies)) + + await crawler.run(['https://httpbingo.org/cookies']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/cookie_management/retry_single_session.py b/docs/guides/code_examples/cookie_management/retry_single_session.py new file mode 100644 index 0000000000..37197fe0c6 --- /dev/null +++ b/docs/guides/code_examples/cookie_management/retry_single_session.py @@ -0,0 +1,41 @@ +import asyncio +from datetime import timedelta + +from crawlee.crawlers import HttpCrawler, HttpCrawlingContext +from crawlee.sessions import SessionPool + + +async def main() -> None: + crawler = HttpCrawler( + # With a single session, don't burn retries trying to rotate to another one. + max_session_rotations=0, + session_pool=SessionPool( + # A single session, so every attempt runs on the same one. + max_pool_size=1, + create_session_settings={ + # Keep the single session alive for the whole crawl. + 'max_usage_count': 999_999, + 'max_age': timedelta(hours=999_999), + 'max_error_score': 100, + }, + ), + ) + + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + if context.session is None: + return + + # First attempt: establish cookies, then raise to trigger a retry. + if not context.session.cookies.get_cookies_as_dicts(): + await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') + raise RuntimeError('retry with cookies') + + # The retry runs on the same session, so the cookies are still here. + context.log.info(f'Cookies: {context.session.cookies.get_cookies_as_dicts()}') + + await crawler.run(['https://httpbingo.org/cookies']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/cookie_management.mdx b/docs/guides/cookie_management.mdx new file mode 100644 index 0000000000..6f023c49a6 --- /dev/null +++ b/docs/guides/cookie_management.mdx @@ -0,0 +1,106 @@ +--- +id: cookie-management +title: Cookie management +description: How to read, set, and persist cookies across requests, retries, and runs in Crawlee. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ReadWriteCookies from '!!raw-loader!roa-loader!./code_examples/cookie_management/read_write_cookies.py'; +import InitialCookies from '!!raw-loader!roa-loader!./code_examples/cookie_management/initial_cookies.py'; +import PlaywrightCookies from '!!raw-loader!roa-loader!./code_examples/cookie_management/playwright_cookies.py'; +import RetrySingleSession from '!!raw-loader!roa-loader!./code_examples/cookie_management/retry_single_session.py'; +import RetryPinnedSession from '!!raw-loader!roa-loader!./code_examples/cookie_management/retry_pinned_session.py'; +import RetryRestoreCookies from '!!raw-loader!roa-loader!./code_examples/cookie_management/retry_restore_cookies.py'; +import PersistCookies from '!!raw-loader!roa-loader!./code_examples/cookie_management/persist_cookies.py'; + +Cookies carry login state, consent flags, CSRF tokens, and other per-user data that a site expects to see on every request. + +In Crawlee, cookies are stored on the `Session` that handles a request, in a `SessionCookies` jar. When a request runs on a session, the HTTP client sends the session's cookies with the request and writes any `Set-Cookie` from the response back onto the same jar. The next request on that session carries the updated cookies. + +Cookies follow the session, not the URL. Two requests share cookies when they run on the same session. For how sessions are created and rotated, see the [Session management](./session-management) guide. + +This guide covers how to read and set cookies, seed them on new sessions, handle them with `PlaywrightCrawler`, and keep them across retries and runs. + +## Reading and setting cookies + +Access the jar through `context.session.cookies`. Use `set` to add a cookie, iterate the jar to read all cookies, and index by name to read one value. + + + {ReadWriteCookies} + + +:::note + +`context.session` is `None` only when the crawler runs without a session pool (`use_session_pool` set to `False`). With the default pool it's always set. The examples guard with `if context.session is None: return` to cover that case. + +::: + +A cookie has a `name`, a `value`, and optional parameters such as `domain`. If a cookie has no `domain`, it applies to any domain. For the full set of parameters, see `CookieParam`. To dump the whole jar, use `get_cookies_as_dicts()`, and to load several cookies at once, use `set_cookies()`. + +## Seeding cookies on new sessions + +To start every session with a known set of cookies, for example a consent flag or a pre-obtained token, pass them through `create_session_settings` on the `SessionPool`. Each new session in the pool is created with these cookies already in its jar. + + + {InitialCookies} + + +## Cookies with PlaywrightCrawler + +`PlaywrightCrawler` keeps the session jar and the browser context in sync. Before navigation, the session's cookies are loaded into the browser context. After the handler returns, cookies from the browser context are merged back onto the session. The merge captures whatever the page picked up through `Set-Cookie`, JavaScript, or redirects. + +The first request receives a cookie in the browser. Crawlee merges it onto the session, so a later request on the same session sees it. + + + {PlaywrightCookies} + + +By default all pages share one browser context, so cookies from different sessions can mix in it. To keep each session's cookies isolated, set `use_incognito_pages` to `True`, which gives each page its own context. + +## Keeping cookies across retries + +By default, a request that isn't pinned to a session draws a new random session from the pool on every attempt. When a handler establishes cookies and then raises to trigger a retry, the retry runs on a different, cookie-less session. The cookies aren't lost. They're still on the first session. The retry just isn't using it. + +Establish the cookies with `send_request`. It runs on the current request's session, so cookies the server sets on that call land on `context.session`. An HTTP call made outside Crawlee doesn't touch the session, so those cookies never reach it. + +There are three ways to carry the cookies over. + +### Use a single-session pool + +Drops session rotation. One session for the whole crawl, so cookies set on the first attempt are still there on the retry. The raise-to-retry flow stays unchanged. It's the simplest fix, and it suits sites you crawl as a single identity. + + + {RetrySingleSession} + + +### Pin the request to its session + +Keeps rotation for the rest of the crawl. Instead of raising, create a new request for the same URL with its session bound through the `session_id` parameter of `Request.from_url`. Setting `always_enqueue=True` re-queues the URL despite deduplication. + + + {RetryPinnedSession} + + +### Re-apply cookies in a pre-navigation hook + +Keeps rotation and the raise-to-retry flow, and shares the cookies across every session. Snapshot the cookies after the setup step into the crawler-wide `use_state` store. Then re-apply them onto whichever session handles the request in a `pre_navigation_hook`, which runs before navigation and receives `context.session`. + +Use it when the cookies belong to the whole crawl rather than one request, for example a consent cookie you obtain once. + + + {RetryRestoreCookies} + + +## Persisting cookies across runs + +The session pool can persist its state, including each session's cookies, to a `KeyValueStore`. Persistence is off by default. Enable it with `persistence_enabled` set to `True` on the `SessionPool`, and give the store a name with `persist_state_kvs_name`. Persistence across restarts needs a named store. An unnamed one is purged on start, so cookies silently won't survive a restart. + +To see restoration in action, define the pool outside the crawler and read a session from it before the run. On the first run the jar is empty. On later runs the cookies from the previous run are already there. + + + {PersistCookies} + + +With persistence enabled, a login you performed on a previous run can carry over, so you skip re-authenticating on every start. For a full walkthrough of logging in, see the [Logging in with a crawler](./logging-in-with-a-crawler) guide. diff --git a/docs/guides/crawler_login.mdx b/docs/guides/crawler_login.mdx index 3632584d48..3dcdc7c72d 100644 --- a/docs/guides/crawler_login.mdx +++ b/docs/guides/crawler_login.mdx @@ -14,7 +14,7 @@ Many websites require authentication to access their content. This guide demonst ## Session management for authentication -When implementing authentication, you'll typically want to maintain the same `Session` throughout your crawl to preserve login state. This requires proper configuration of the `SessionPool`. For more details, see our [session management guide](./session-management). +When implementing authentication, you'll typically want to maintain the same `Session` throughout your crawl to preserve login state. This requires proper configuration of the `SessionPool`. For more details, see our [session management guide](./session-management). To work with the cookies themselves, see [Cookie management](./cookie-management). If your use case requires multiple authenticated sessions with different credentials, you can: diff --git a/docs/guides/http_headers.mdx b/docs/guides/http_headers.mdx index 6ef7c864be..06f4566efa 100644 --- a/docs/guides/http_headers.mdx +++ b/docs/guides/http_headers.mdx @@ -41,7 +41,7 @@ These headers tell the server what the client can handle and the server uses the ### Authentication and stateful headers -`Cookie` carries session and login state. Crawlee manages cookies through [sessions](./session-management), so you rarely set this one by hand. +`Cookie` carries session and login state. Crawlee manages cookies through [sessions](./session-management), so you rarely set this one by hand. For details, see [Cookie management](./cookie-management). `Authorization` carries credentials, such as a bearer token or basic auth. APIs commonly require it. Set it on the request when the target needs authenticated access. Treat its value as a secret, and don't send it through a [proxy you don't control](./security-of-web-scraping#untrusted-proxies). diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx index 493715d301..772240808d 100644 --- a/docs/guides/session_management.mdx +++ b/docs/guides/session_management.mdx @@ -24,7 +24,7 @@ Additionally, it enables storing information tied to specific IP addresses, such Finally, it ensures even IP address rotation by randomly selecting sessions. This helps prevent overuse of a limited pool of available IPs, reducing the risk of IP bans and enhancing the efficiency of your scraper. -For more details on configuring proxies, refer to the [Proxy management](./proxy-management) guide. +For more details on configuring proxies, refer to the [Proxy management](./proxy-management) guide. To read, set, and persist cookies, see [Cookie management](./cookie-management). Now, let's explore examples of how to use the `SessionPool` in different scenarios: