Python is a high-level, easy-to-read programming language that lets you write powerful software quickly with clear, expressive code.
\n
\n
\n
\n
\n
\n
\n\n
\n
\n
\n
\n\n\n\n
\n
\n
\n
\n
\n
\n
\n
\n\n
\n\n
\n
\n
\n\n
\n
\n
\n\n
\n\n\n
\n
\n
\n
\n
ChatGPT can make mistakes. Check important info.
\n
\n
\n
\n
\n\n
\n
\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
- "answer_text": "Python is a high-level, easy-to-read programming language that lets you write powerful software quickly with clear, expressive code.",
- "links_attached": null,
- "citations": null,
- "recommendations": [],
- "country": "US",
- "is_map": false,
- "references": [],
- "shopping": [],
- "shopping_visible": false,
- "index": null,
- "answer_text_markdown": "Python is a high-level, easy-to-read programming language that lets you write powerful software quickly with clear, expressive code.",
- "web_search_triggered": false,
- "additional_prompt": null,
- "additional_answer_text": null,
- "map": null,
- "search_sources": [],
- "response_raw": "[{\"p\":\"\",\"o\":\"add\",\"v\":{\"message\":{\"id\":\"743621fa-eb8d-4ab8-9425-d06bb28abbb6\",\"author\":{\"role\":\"system\",\"name\":null,\"metadata\":{}},\"create_time\":null,\"update_time\":null,\"content\":{\"content_type\":\"text\",\"parts\":[\"\"]},\"status\":\"finished_successfully\",\"end_turn\":true,\"weight\":0,\"metadata\":{\"is_visually_hidden_from_conversation\":true,\"model_switcher_deny\":[]},\"recipient\":\"all\",\"channel\":null},\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\",\"error\":null},\"c\":0},{\"v\":{\"message\":{\"id\":\"c7607fa3-c93f-4449-863b-a5592d98e2c3\",\"author\":{\"role\":\"user\",\"name\":null,\"metadata\":{}},\"create_time\":1763658097.325,\"update_time\":null,\"content\":{\"content_type\":\"text\",\"parts\":[\"Explain Python in one sentence\"]},\"status\":\"finished_successfully\",\"end_turn\":null,\"weight\":1,\"metadata\":{\"system_hints\":[],\"request_id\":\"ac58d482-927b-4d2b-ae25-d3f6973af414\",\"message_source\":\"instant-query\",\"turn_exchange_id\":\"0bd313d0-3fb0-4b8e-b029-734a99b893ce\",\"timestamp_\":\"absolute\",\"model_switcher_deny\":[]},\"recipient\":\"all\",\"channel\":null},\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\",\"error\":null},\"c\":1},{\"type\":\"input_message\",\"input_message\":{\"id\":\"c7607fa3-c93f-4449-863b-a5592d98e2c3\",\"author\":{\"role\":\"user\",\"name\":null,\"metadata\":{}},\"create_time\":1763658097.325,\"update_time\":null,\"content\":{\"content_type\":\"text\",\"parts\":[\"Explain Python in one sentence\"]},\"status\":\"finished_successfully\",\"end_turn\":null,\"weight\":1,\"metadata\":{\"system_hints\":[],\"request_id\":\"ac58d482-927b-4d2b-ae25-d3f6973af414\",\"message_source\":\"instant-query\",\"turn_exchange_id\":\"0bd313d0-3fb0-4b8e-b029-734a99b893ce\",\"useragent\":{\"client_type\":\"web\",\"is_mobile\":false,\"is_mobile_app\":false,\"is_desktop_app\":false,\"is_native_app\":false,\"is_native_app_apple\":false,\"is_mobile_app_ios\":false,\"is_desktop_app_macos\":false,\"is_aura_app_macos\":false,\"is_aura_web\":false,\"is_sora_ios\":false,\"is_agora_ios\":false,\"is_agora_android\":false,\"is_desktop_app_windows\":false,\"is_electron_app\":false,\"is_mobile_app_android\":false,\"is_mobile_web\":false,\"is_mobile_web_ios\":false,\"is_mobile_web_android\":false,\"is_ios\":false,\"is_android\":false,\"is_chatgpt_client\":false,\"is_sora_client\":false,\"is_agora_client\":false,\"is_browserbased_app\":true,\"is_chatgpt_api\":false,\"is_slack\":false,\"is_chatkit_web\":false,\"is_chatkit_synthetic\":false,\"is_kakao_talk\":false,\"app_version\":null,\"build_number\":null,\"user_agent\":\"mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/142.0.0.0 safari/537.36\",\"app_environment\":null,\"os_version\":null,\"device_model\":null,\"user_client_type\":\"desktop_web\"},\"timestamp_\":\"absolute\",\"paragen_stream_type\":\"default\",\"parent_id\":\"743621fa-eb8d-4ab8-9425-d06bb28abbb6\"},\"recipient\":\"all\",\"channel\":null},\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\"},{\"v\":{\"message\":{\"id\":\"c288c4ba-2d36-4349-b9bf-f3b57337e2db\",\"author\":{\"role\":\"assistant\",\"name\":null,\"metadata\":{}},\"create_time\":1763658101.956405,\"update_time\":1763658102.069259,\"content\":{\"content_type\":\"text\",\"parts\":[\"\"]},\"status\":\"in_progress\",\"end_turn\":null,\"weight\":1,\"metadata\":{\"citations\":[],\"content_references\":[],\"request_id\":\"ac58d482-927b-4d2b-ae25-d3f6973af414\",\"message_type\":\"next\",\"model_slug\":\"gpt-5-1\",\"default_model_slug\":\"auto\",\"parent_id\":\"c7607fa3-c93f-4449-863b-a5592d98e2c3\",\"turn_exchange_id\":\"0bd313d0-3fb0-4b8e-b029-734a99b893ce\",\"timestamp_\":\"absolute\",\"model_switcher_deny\":[]},\"recipient\":\"all\",\"channel\":\"final\"},\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\",\"error\":null},\"c\":2},{\"type\":\"server_ste_metadata\",\"metadata\":{\"conduit_prewarmed\":false,\"fast_convo\":true,\"warmup_state\":\"cold\",\"is_first_turn\":true,\"model_slug\":\"gpt-5-1\",\"did_auto_switch_to_reasoning\":false,\"auto_switcher_race_winner\":\"autoswitcher\",\"is_autoswitcher_enabled\":true,\"is_search\":null,\"did_prompt_contain_image\":false,\"message_id\":\"c288c4ba-2d36-4349-b9bf-f3b57337e2db\",\"request_id\":\"ac58d482-927b-4d2b-ae25-d3f6973af414\"},\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\"},{\"type\":\"message_marker\",\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\",\"message_id\":\"c288c4ba-2d36-4349-b9bf-f3b57337e2db\",\"marker\":\"user_visible_token\",\"event\":\"first\"},{\"o\":\"patch\",\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658102.077773},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.105858},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\"Python is\"}]},{\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658102.134441},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.195202},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\" a high-\"}]},{\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658102.267426},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.295866},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\"level, easy\"}]},{\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658102.445245},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.513697},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\"-to-read programming language\"}]},{\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658102.699152},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.72003},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\" that lets you write powerful software quickly\"}]},{\"v\":[{\"p\":\"/message/create_time\",\"o\":\"replace\",\"v\":1763658101.956405},{\"p\":\"/message/update_time\",\"o\":\"replace\",\"v\":1763658102.852512},{\"p\":\"/message/content/parts/0\",\"o\":\"append\",\"v\":\" with clear, expressive code.\"},{\"p\":\"/message/status\",\"o\":\"replace\",\"v\":\"finished_successfully\"},{\"p\":\"/message/end_turn\",\"o\":\"replace\",\"v\":true},{\"p\":\"/message/metadata\",\"o\":\"append\",\"v\":{\"is_complete\":true,\"finish_details\":{\"type\":\"stop\",\"stop_tokens\":[200002]},\"sonic_classification_result\":{\"latency_ms\":19.449779065325856,\"simple_search_prob\":0.1281321013316676,\"complex_search_prob\":0.00004177866803718204,\"no_search_prob\":0.8718261200002951,\"search_complexity_decision\":\"no_search\",\"search_decision\":false,\"simple_search_threshold\":0,\"complex_search_threshold\":0.4,\"no_search_threshold\":0.12,\"threshold_order\":[\"no_search\",\"complex\",\"simple\"],\"classifier_config_name\":\"sonic_classifier_3cls_ev3\",\"classifier_config\":{\"model_name\":\"snc-pg-sw-3cls-ev3\",\"renderer_name\":\"harmony_v4.0.15_16k_orion_text_only_no_asr_2k_action\",\"force_disabled_rate\":0,\"force_enabled_rate\":0,\"num_messages\":20,\"only_user_messages\":false,\"remove_memory\":true,\"support_mm\":true,\"n_ctx\":2048,\"max_action_length\":4,\"dynamic_set_max_message_size\":false,\"max_message_tokens\":2000,\"append_base_config\":false,\"no_search_token\":\"1\",\"simple_search_token\":\"7\",\"complex_search_token\":\"5\",\"simple_search_threshold\":0,\"complex_search_threshold\":0.4,\"no_search_threshold\":0.12,\"prefetch_threshold\":null,\"force_search_first_turn_threshold\":0.00001,\"threshold_order\":[\"no_search\",\"complex\",\"simple\"],\"passthrough_tool_calls\":null,\"timeout\":1},\"decision_source\":\"classifier\",\"passthrough_tool_names\":[]}}}]},{\"type\":\"message_stream_complete\",\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\"},{\"type\":\"conversation_detail_metadata\",\"banner_info\":null,\"blocked_features\":[],\"model_limits\":[],\"limits_progress\":[{\"feature_name\":\"file_upload\",\"remaining\":3,\"reset_after\":\"2025-11-21T17:01:43.229556+00:00\"}],\"default_model_slug\":\"auto\",\"conversation_id\":\"691f4974-ece8-8330-8906-75b31eccd63a\"}]",
- "answer_section_html": "
\n
\n
\n
\n
\n
\n
Python is a high-level, easy-to-read programming language that lets you write powerful software quickly with clear, expressive code.
\n
\n
\n
\n
\n
\n
\n\n
\n
\n
\n
",
- "model": "gpt-5-1",
- "web_search_query": null,
- "timestamp": "2025-11-20T17:01:52.049Z",
- "input": {
- "url": "https://chatgpt.com/",
- "prompt": "Explain Python in one sentence",
- "country": "US",
- "web_search": false,
- "additional_prompt": ""
- }
- }
-]
\ No newline at end of file
diff --git a/tests/samples/facebook/posts.json b/tests/samples/facebook/posts.json
deleted file mode 100644
index 7a6609d..0000000
--- a/tests/samples/facebook/posts.json
+++ /dev/null
@@ -1,537 +0,0 @@
-[
- {
- "url": "https://www.facebook.com/reel/1178168373700071/",
- "post_id": "1346166837555333",
- "user_url": "https://www.facebook.com/facebook",
- "user_username_raw": "Facebook",
- "content": "While in Nashville for the #FacebookRoadTrip, we caught up with singer-songwriter Kane Brown on everything from golfing in Scotland to reminiscing about his very first tour. Share your own memories from the road to Kane\u2019s Fan Challenge on Facebook using #RoadTripMemoriesChallenge \ud83e\udd20",
- "date_posted": "2025-11-19T20:40:47.000Z",
- "hashtags": [
- "facebookroadtrip"
- ],
- "num_comments": 2093,
- "num_shares": 157,
- "num_likes_type": {
- "type": "Like",
- "num": 6356
- },
- "page_name": "Facebook",
- "profile_id": "100064860875397",
- "page_intro": "Page \u00b7 Internet company",
- "page_category": "Internet company",
- "page_logo": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "page_external_website": "fb.me/HowToContactFB",
- "page_followers": 155000000,
- "page_is_verified": true,
- "attachments": [
- {
- "id": "1178168373700071",
- "type": "Video",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t15.5256-10/584976832_871349975319798_4871365287803428825_n.jpg?stp=dst-jpg_p296x100_tt6&_nc_cat=1&ccb=1-7&_nc_sid=d2b52d&_nc_ohc=gjMY8ZReEDoQ7kNvwGa8N-t&_nc_oc=Adl-ppGoZbPqGT487mkOT_ZyctGC7JXlKIS0zlWBTxZngZZPrwUF6rvTHPARo2g1XuY&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=-3h60myjfLqRmenlXvzYQg&oh=00_AfixTptTxvM9KohUg9LqBxrnSsWUoThZT5WAvCa9gOylXA&oe=69250419",
- "video_length": "60400",
- "attachment_url": "https://www.facebook.com/reel/1178168373700071/",
- "video_url": "https://video.fotp3-2.fna.fbcdn.net/o1/v/t2/f2/m366/AQMaVXPDlqn-RupvW09GASa3Gn4QKH2Vp_N1bpg0NrK0W5MONdKe4jnNJqLIyU9zoaXhUy7vfnWThFUyzmro_cgEuOYaCpFVcuNiXi_K6_EPnA.mp4?_nc_cat=109&_nc_oc=Adk9XFWEXJB9J4dxN_xZQ6g9L9DT1sDIysvNTKyxpB78y5pWs7wYxpo7-edLigPnfZE&_nc_sid=5e9851&_nc_ht=video.fotp3-2.fna.fbcdn.net&_nc_ohc=YHFzhNGXeSgQ7kNvwERY-wD&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5GQUNFQk9PSy4uQzMuNzIwLmRhc2hfaDI2NC1iYXNpYy1nZW4yXzcyMHAiLCJ4cHZfYXNzZXRfaWQiOjE5MTUyMjkyNDkzOTk5MzYsImFzc2V0X2FnZV9kYXlzIjowLCJ2aV91c2VjYXNlX2lkIjoxMDEyMiwiZHVyYXRpb25fcyI6NjAsInVybGdlbl9zb3VyY2UiOiJ3d3cifQ%3D%3D&ccb=17-1&vs=7200846e54bcdebc&_nc_vs=HBksFQIYRWZiX2VwaGVtZXJhbC9CRjQ3QUExRDk3MUU2MDhBNkJGODY1RUQwQUZCMDA4N19tdF8xX3ZpZGVvX2Rhc2hpbml0Lm1wNBUAAsgBEgAVAhhAZmJfcGVybWFuZW50LzA2NEUzQjMwRDVGNDNDOUVFNzI4OENFN0ZFODc0Q0FFX2F1ZGlvX2Rhc2hpbml0Lm1wNBUCAsgBEgAoABgAGwKIB3VzZV9vaWwBMRJwcm9ncmVzc2l2ZV9yZWNpcGUBMRUAACaAspfxgfnmBhUCKAJDMywXQE4zMzMzMzMYGWRhc2hfaDI2NC1iYXNpYy1nZW4yXzcyMHARAHUCZZSeAQA&_nc_gid=-3h60myjfLqRmenlXvzYQg&_nc_zt=28&oh=00_AfimffoprOlAs92pqZdC2KPErVR0HJTFRLSaUoxCdzEL6g&oe=69251CA3&bitrate=1997814&tag=dash_h264-basic-gen2_720p"
- }
- ],
- "post_external_image": null,
- "page_url": "https://www.facebook.com/facebook",
- "header_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/513094825_10164819146606729_8444440187994304660_n.jpg?stp=dst-jpg_s960x960_tt6&_nc_cat=110&ccb=1-7&_nc_sid=cc71e4&_nc_ohc=VsiHP2aGf3MQ7kNvwFTV3XC&_nc_oc=AdkylD-RY8FvW2JntucYN4H7R89r36f2Bd_ogoTze8GT_dAnJbCu-RKxVkl6QfZsw9I&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_Afg6j5-JjOxy77BdW2zEv1Zqhw6_y8xb4Z0ee6b8zX22fA&oe=6925133D",
- "avatar_image_url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "profile_handle": "facebook",
- "is_sponsored": false,
- "shortcode": "1346166837555333",
- "video_view_count": 48896,
- "likes": 8018,
- "post_type": "Reel",
- "following": null,
- "link_description_text": null,
- "count_reactions_type": [
- {
- "type": "Like",
- "reaction_count": 6356
- },
- {
- "type": "Love",
- "reaction_count": 1354
- },
- {
- "type": "Care",
- "reaction_count": 246
- },
- {
- "type": "Wow",
- "reaction_count": 46
- },
- {
- "type": "Haha",
- "reaction_count": 8
- },
- {
- "type": "Sad",
- "reaction_count": 5
- },
- {
- "type": "Angry",
- "reaction_count": 3
- }
- ],
- "is_page": true,
- "page_phone": null,
- "page_email": null,
- "page_creation_time": "2007-11-07T00:00:00.000Z",
- "page_reviews_score": null,
- "page_reviewers_amount": null,
- "page_price_range": null,
- "about": [
- {
- "type": "INFLUENCER CATEGORY",
- "value": "Page \u00b7 Internet company",
- "link": null
- },
- {
- "type": "WEBSITE",
- "value": "fb.me/HowToContactFB",
- "link": "https://fb.me/HowToContactFB"
- }
- ],
- "active_ads_urls": [],
- "delegate_page_id": "20531316728",
- "privacy_and_legal_info": null,
- "timestamp": "2025-11-20T16:55:55.934Z",
- "input": {
- "url": "https://www.facebook.com/facebook",
- "num_of_posts": 5,
- "start_date": "",
- "end_date": ""
- }
- },
- {
- "url": "https://www.facebook.com/facebook/posts/pfbid02o9kd9bePA6C6EdPHyPEUsKGDeM9QmJ4EPY7BdZnUzJKe9EHDZkkf3AtCNd3ZxeU4l",
- "post_id": "1346025967569420",
- "user_url": "https://www.facebook.com/facebook",
- "user_username_raw": "Facebook",
- "content": "Hey, Music City! We\u2019re headed to seven US cities on the #FacebookRoadTrip to bring the Facebook vibes to all our friends IRL. Check out all the fun we had in Nashville and be sure to join us at our *last* stop on the tour in New York City next month!",
- "date_posted": "2025-11-19T16:59:27.000Z",
- "hashtags": [
- "facebookroadtrip"
- ],
- "num_comments": 8757,
- "num_shares": 573,
- "num_likes_type": {
- "type": "Like",
- "num": 23285
- },
- "page_name": "Facebook",
- "profile_id": "100064860875397",
- "page_intro": "Page \u00b7 Internet company",
- "page_category": "Internet company",
- "page_logo": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "page_external_website": "fb.me/HowToContactFB",
- "page_followers": 155000000,
- "page_is_verified": true,
- "attachments": [
- {
- "id": "1346022090903141",
- "type": "Photo",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/585669351_1346026050902745_7640051638980346272_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=GpHH57YINDsQ7kNvwHYa6Am&_nc_oc=AdkTbnmoGEgm3PNARgBirW9QhrL-v4SxrJVRTM-zv5exYSemUW6CN_UpLonpZfll_iI&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfipsOuKQ3ZfBQCUaffMcQI89jYZXEgek3QtAdCuOrhXkw&oe=69252EA5",
- "attachment_url": "https://www.facebook.com/photo.php?fbid=1346022090903141&set=a.1272781121560572&type=3",
- "video_url": null
- },
- {
- "id": "1346022140903136",
- "type": "Photo",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/584614169_1346026080902742_8497372545534067199_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=rx_aej1IiugQ7kNvwFjW98p&_nc_oc=AdkxB7s6iOJSXyeOjmnGy9y_RSex-qScBAsxd7jQ-zY2Lb6vbMB4RmdOxNv2VK5RGKs&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfhctYoo9bNwtA_6Xq7at8Z0K9Lk1EzuycXCOdvGsmubPw&oe=69250B3B",
- "attachment_url": "https://www.facebook.com/photo.php?fbid=1346022140903136&set=a.1272781121560572&type=3",
- "video_url": null
- },
- {
- "id": "1346022154236468",
- "type": "Photo",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/585343367_1346026084236075_767938696844464465_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=gcgvljl_EHEQ7kNvwH6jsjF&_nc_oc=AdkDcHIJaoW90iO8TdvguiMjpIjgyChIj8ykD4evRmWpU0X9QOoa11sg6cfSPkk2VUs&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfhT6Fbu0LyofJXR2ZhNX4mAOwkN_2LPJda4Oy5mAK46zw&oe=69251D3C",
- "attachment_url": "https://www.facebook.com/photo.php?fbid=1346022154236468&set=a.1272781121560572&type=3",
- "video_url": null
- },
- {
- "id": "1346022194236464",
- "type": "Photo",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/584731919_1346026097569407_5936004192315395883_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=HArPchLOCFIQ7kNvwEqOT25&_nc_oc=AdlpNnL2wTM4iuXkPlFZRFCKoPjJPtJ5rJIOBNCNQjshM-QRRfisFeJgWEThuHDil14&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfhwMz0UnQCkilZ4FKtjnXwj-UhdLQPfiLM99t_rwx4kug&oe=69250D3D",
- "attachment_url": "https://www.facebook.com/photo.php?fbid=1346022194236464&set=a.1272781121560572&type=3",
- "video_url": null
- },
- {
- "id": "1346022104236473",
- "type": "Photo",
- "url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/587247300_1346026057569411_1976402081820657581_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=P9XJp6BUEFAQ7kNvwFxdU8-&_nc_oc=AdnAAmB317anVCSGf6SwjCWxoV3AYXf5GE2jauJbNUOMNMnZPYZX8EmBsO-qJcc9CtM&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfhYSPTWNC6xMQcTtQl8_YAnlOQHIx8sK-yTWjL0cL3uRQ&oe=692526FC",
- "attachment_url": "https://www.facebook.com/photo.php?fbid=1346022104236473&set=a.1272781121560572&type=3",
- "video_url": null
- }
- ],
- "post_external_image": null,
- "page_url": "https://www.facebook.com/facebook",
- "header_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/513094825_10164819146606729_8444440187994304660_n.jpg?stp=dst-jpg_s960x960_tt6&_nc_cat=110&ccb=1-7&_nc_sid=cc71e4&_nc_ohc=VsiHP2aGf3MQ7kNvwFTV3XC&_nc_oc=AdkylD-RY8FvW2JntucYN4H7R89r36f2Bd_ogoTze8GT_dAnJbCu-RKxVkl6QfZsw9I&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_Afg6j5-JjOxy77BdW2zEv1Zqhw6_y8xb4Z0ee6b8zX22fA&oe=6925133D",
- "avatar_image_url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "profile_handle": "facebook",
- "is_sponsored": false,
- "shortcode": "1346025967569420",
- "likes": 30321,
- "post_image": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-6/585669351_1346026050902745_7640051638980346272_n.jpg?_nc_cat=1&ccb=1-7&_nc_sid=f727a1&_nc_ohc=GpHH57YINDsQ7kNvwHYa6Am&_nc_oc=AdkTbnmoGEgm3PNARgBirW9QhrL-v4SxrJVRTM-zv5exYSemUW6CN_UpLonpZfll_iI&_nc_zt=23&_nc_ht=scontent.fotp3-3.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfipsOuKQ3ZfBQCUaffMcQI89jYZXEgek3QtAdCuOrhXkw&oe=69252EA5",
- "post_type": "Post",
- "following": null,
- "link_description_text": null,
- "count_reactions_type": [
- {
- "type": "Like",
- "reaction_count": 23285
- },
- {
- "type": "Love",
- "reaction_count": 5845
- },
- {
- "type": "Care",
- "reaction_count": 889
- },
- {
- "type": "Wow",
- "reaction_count": 229
- },
- {
- "type": "Haha",
- "reaction_count": 56
- },
- {
- "type": "Sad",
- "reaction_count": 9
- },
- {
- "type": "Angry",
- "reaction_count": 8
- }
- ],
- "is_page": true,
- "page_phone": null,
- "page_email": null,
- "page_creation_time": "2007-11-07T00:00:00.000Z",
- "page_reviews_score": null,
- "page_reviewers_amount": null,
- "page_price_range": null,
- "about": [
- {
- "type": "INFLUENCER CATEGORY",
- "value": "Page \u00b7 Internet company",
- "link": null
- },
- {
- "type": "WEBSITE",
- "value": "fb.me/HowToContactFB",
- "link": "https://fb.me/HowToContactFB"
- }
- ],
- "active_ads_urls": [],
- "delegate_page_id": "20531316728",
- "privacy_and_legal_info": null,
- "timestamp": "2025-11-20T16:55:55.934Z",
- "input": {
- "url": "https://www.facebook.com/facebook",
- "num_of_posts": 5,
- "start_date": "",
- "end_date": ""
- }
- },
- {
- "url": "https://www.facebook.com/facebook/posts/pfbid02nHWsd8pxGMmvvEEEyv2JKMCKK9g74F35PceVr7onVQq7dDx9PddoRLw6GndboRCLl",
- "post_id": "1345095954329088",
- "user_url": "https://www.facebook.com/facebook",
- "user_username_raw": "Facebook",
- "content": "Put a finger down if you\u2019re currently spiraling after liking your crush\u2019s story\u2026",
- "date_posted": "2025-11-18T17:00:00.000Z",
- "num_comments": 5303,
- "num_shares": 392,
- "num_likes_type": {
- "type": "Like",
- "num": 18443
- },
- "page_name": "Facebook",
- "profile_id": "100064860875397",
- "page_intro": "Page \u00b7 Internet company",
- "page_category": "Internet company",
- "page_logo": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "page_external_website": "fb.me/HowToContactFB",
- "page_followers": 155000000,
- "page_is_verified": true,
- "post_external_image": null,
- "page_url": "https://www.facebook.com/facebook",
- "header_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/513094825_10164819146606729_8444440187994304660_n.jpg?stp=dst-jpg_s960x960_tt6&_nc_cat=110&ccb=1-7&_nc_sid=cc71e4&_nc_ohc=VsiHP2aGf3MQ7kNvwFTV3XC&_nc_oc=AdkylD-RY8FvW2JntucYN4H7R89r36f2Bd_ogoTze8GT_dAnJbCu-RKxVkl6QfZsw9I&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_Afg6j5-JjOxy77BdW2zEv1Zqhw6_y8xb4Z0ee6b8zX22fA&oe=6925133D",
- "avatar_image_url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "profile_handle": "facebook",
- "is_sponsored": false,
- "shortcode": "1345095954329088",
- "likes": 23613,
- "post_type": "Post",
- "following": null,
- "link_description_text": null,
- "count_reactions_type": [
- {
- "type": "Like",
- "reaction_count": 18443
- },
- {
- "type": "Love",
- "reaction_count": 3678
- },
- {
- "type": "Haha",
- "reaction_count": 802
- },
- {
- "type": "Care",
- "reaction_count": 550
- },
- {
- "type": "Wow",
- "reaction_count": 85
- },
- {
- "type": "Sad",
- "reaction_count": 30
- },
- {
- "type": "Angry",
- "reaction_count": 25
- }
- ],
- "is_page": true,
- "page_phone": null,
- "page_email": null,
- "page_creation_time": "2007-11-07T00:00:00.000Z",
- "page_reviews_score": null,
- "page_reviewers_amount": null,
- "page_price_range": null,
- "about": [
- {
- "type": "INFLUENCER CATEGORY",
- "value": "Page \u00b7 Internet company",
- "link": null
- },
- {
- "type": "WEBSITE",
- "value": "fb.me/HowToContactFB",
- "link": "https://fb.me/HowToContactFB"
- }
- ],
- "active_ads_urls": [],
- "delegate_page_id": "20531316728",
- "privacy_and_legal_info": null,
- "timestamp": "2025-11-20T16:55:55.934Z",
- "input": {
- "url": "https://www.facebook.com/facebook",
- "num_of_posts": 5,
- "start_date": "",
- "end_date": ""
- }
- },
- {
- "url": "https://www.facebook.com/reel/1381683193563154/",
- "post_id": "1344308637741153",
- "user_url": "https://www.facebook.com/facebook",
- "user_username_raw": "Facebook",
- "content": "This reel is your urgent reminder that soup szn has arrived \ud83e\udd24\n\nVideo by Essen Paradies",
- "date_posted": "2025-11-17T21:59:55.000Z",
- "num_comments": 3091,
- "num_shares": 2368,
- "num_likes_type": {
- "type": "Like",
- "num": 18297
- },
- "page_name": "Facebook",
- "profile_id": "100064860875397",
- "page_intro": "Page \u00b7 Internet company",
- "page_category": "Internet company",
- "page_logo": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "page_external_website": "fb.me/HowToContactFB",
- "page_followers": 155000000,
- "page_is_verified": true,
- "attachments": [
- {
- "id": "1381683193563154",
- "type": "Video",
- "url": "https://scontent.fotp3-4.fna.fbcdn.net/v/t15.5256-10/583966455_33094466116867804_7048232568839350902_n.jpg?stp=dst-jpg_p296x100_tt6&_nc_cat=108&ccb=1-7&_nc_sid=d2b52d&_nc_ohc=lPrKUi3BRIwQ7kNvwGJ3H9R&_nc_oc=AdkQQNfEqT-WjYi-Y2_88OyKeSJLKLB0KgoAq5zfwF592KRG6Vwnbj8xjbp-HylnXcM&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&oh=00_AfjPtucBTof3JQOP7l8yI9ej1mrEuhUFs-85HoE1mPillw&oe=69251884",
- "video_length": "24700",
- "attachment_url": "https://www.facebook.com/reel/1381683193563154/",
- "video_url": "https://video.fotp3-2.fna.fbcdn.net/o1/v/t2/f2/m366/AQOW0kYCUDer2UeIrz3h4fMr4dfT80_dIwF6WxM6Cru0cYzWYP13O4FE8-0kh3UBV0Iq1X6mGfUxYhADV8hFnKrv-5v5zoF7BhmmyA4tnnsyoA.mp4?_nc_cat=105&_nc_oc=AdnvPq5uGqIhohUE2ZR4lUyI6-amonnjO3IBNPthwJpOqiUMszG9WktmU3LKElFqONc&_nc_sid=5e9851&_nc_ht=video.fotp3-2.fna.fbcdn.net&_nc_ohc=E3a9CqQXQhMQ7kNvwFgFqCF&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5GQUNFQk9PSy4uQzMuNzIwLmRhc2hfaDI2NC1iYXNpYy1nZW4yXzcyMHAiLCJ4cHZfYXNzZXRfaWQiOjgyMDg1MjYxNzIyMDMwMiwiYXNzZXRfYWdlX2RheXMiOjMsInZpX3VzZWNhc2VfaWQiOjEwMTIyLCJkdXJhdGlvbl9zIjoyNCwidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&vs=81c995631c3d8b89&_nc_vs=HBksFQIYRWZiX2VwaGVtZXJhbC9EQjRCQjIyMUYwRkQzODg1NTA1MzFEMDUyQ0IzNTZBQl9tdF8xX3ZpZGVvX2Rhc2hpbml0Lm1wNBUAAsgBEgAVAhhAZmJfcGVybWFuZW50L0Y3NDM1NkExQTYzMUJBMzFCMUE3QTY5QzlFRUIyMjlDX2F1ZGlvX2Rhc2hpbml0Lm1wNBUCAsgBEgAoABgAGwKIB3VzZV9vaWwBMRJwcm9ncmVzc2l2ZV9yZWNpcGUBMRUAACacw8zK9KP1AhUCKAJDMywXQDizMzMzMzMYGWRhc2hfaDI2NC1iYXNpYy1nZW4yXzcyMHARAHUCZZSeAQA&_nc_gid=_MKGgoF8MSZeS1IDEI0gtw&_nc_zt=28&oh=00_AfjXRi9v9QrT_Cjm3Cg1-gOcU5fPalkt147GYfZyvoS_rQ&oe=6925116B&bitrate=2751266&tag=dash_h264-basic-gen2_720p"
- }
- ],
- "post_external_image": null,
- "page_url": "https://www.facebook.com/facebook",
- "header_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/513094825_10164819146606729_8444440187994304660_n.jpg?stp=dst-jpg_s960x960_tt6&_nc_cat=110&ccb=1-7&_nc_sid=cc71e4&_nc_ohc=VsiHP2aGf3MQ7kNvwFTV3XC&_nc_oc=AdkylD-RY8FvW2JntucYN4H7R89r36f2Bd_ogoTze8GT_dAnJbCu-RKxVkl6QfZsw9I&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_Afg6j5-JjOxy77BdW2zEv1Zqhw6_y8xb4Z0ee6b8zX22fA&oe=6925133D",
- "avatar_image_url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "profile_handle": "facebook",
- "is_sponsored": false,
- "shortcode": "1344308637741153",
- "video_view_count": 1348545,
- "likes": 22573,
- "post_type": "Reel",
- "following": null,
- "link_description_text": null,
- "count_reactions_type": [
- {
- "type": "Like",
- "reaction_count": 18297
- },
- {
- "type": "Love",
- "reaction_count": 3571
- },
- {
- "type": "Wow",
- "reaction_count": 360
- },
- {
- "type": "Care",
- "reaction_count": 289
- },
- {
- "type": "Haha",
- "reaction_count": 34
- },
- {
- "type": "Sad",
- "reaction_count": 12
- },
- {
- "type": "Angry",
- "reaction_count": 10
- }
- ],
- "is_page": true,
- "page_phone": null,
- "page_email": null,
- "page_creation_time": "2007-11-07T00:00:00.000Z",
- "page_reviews_score": null,
- "page_reviewers_amount": null,
- "page_price_range": null,
- "about": [
- {
- "type": "INFLUENCER CATEGORY",
- "value": "Page \u00b7 Internet company",
- "link": null
- },
- {
- "type": "WEBSITE",
- "value": "fb.me/HowToContactFB",
- "link": "https://fb.me/HowToContactFB"
- }
- ],
- "active_ads_urls": [],
- "delegate_page_id": "20531316728",
- "privacy_and_legal_info": null,
- "timestamp": "2025-11-20T16:55:55.934Z",
- "input": {
- "url": "https://www.facebook.com/facebook",
- "num_of_posts": 5,
- "start_date": "",
- "end_date": ""
- }
- },
- {
- "url": "https://www.facebook.com/facebook/posts/pfbid0cjvy6GcddaRhymuiwpnXDdvaVyRy7ZzTT5N8zvKJXEGvvTb3bFmKne6H6J8aVYvol",
- "post_id": "1344226454416038",
- "user_url": "https://www.facebook.com/facebook",
- "user_username_raw": "Facebook",
- "content": "\u2018Tis the season to ask Meta AI for yummy baking recipes\n\nMade with Meta AI",
- "date_posted": "2025-11-17T19:59:56.000Z",
- "num_comments": 3456,
- "num_shares": 372,
- "num_likes_type": {
- "type": "Like",
- "num": 9601
- },
- "page_name": "Facebook",
- "profile_id": "100064860875397",
- "page_intro": "Page \u00b7 Internet company",
- "page_category": "Internet company",
- "page_logo": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "page_external_website": "fb.me/HowToContactFB",
- "page_followers": 155000000,
- "page_is_verified": true,
- "attachments": [
- {
- "id": "1344102401095110",
- "type": "Photo",
- "url": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/583535523_1344102404428443_764020420504838959_n.jpg?stp=dst-jpg_p526x296_tt6&_nc_cat=110&ccb=1-7&_nc_sid=833d8c&_nc_ohc=UwXx_w-yY-4Q7kNvwHI92JR&_nc_oc=AdnIBQD97VrFs4cwsrObV-NB13U0OFu83IukV4n07p9jKd_bGA_GI5OpoufEK8BkeeA&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=WHg8XZNKQkBGkIvCnGF3kQ&oh=00_AfgnWX67Rke8m83S2TxFla4c1rJdRxMThFbqBT1O7eGyrg&oe=69250449",
- "video_url": null
- }
- ],
- "post_external_image": null,
- "page_url": "https://www.facebook.com/facebook",
- "header_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/513094825_10164819146606729_8444440187994304660_n.jpg?stp=dst-jpg_s960x960_tt6&_nc_cat=110&ccb=1-7&_nc_sid=cc71e4&_nc_ohc=VsiHP2aGf3MQ7kNvwFTV3XC&_nc_oc=AdkylD-RY8FvW2JntucYN4H7R89r36f2Bd_ogoTze8GT_dAnJbCu-RKxVkl6QfZsw9I&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_Afg6j5-JjOxy77BdW2zEv1Zqhw6_y8xb4Z0ee6b8zX22fA&oe=6925133D",
- "avatar_image_url": "https://scontent.fotp3-3.fna.fbcdn.net/v/t39.30808-1/380700650_10162533193146729_2379134611963304810_n.jpg?stp=dst-jpg_s200x200_tt6&_nc_cat=1&ccb=1-7&_nc_sid=2d3e12&_nc_ohc=oDCg5qbKk18Q7kNvwH1omta&_nc_oc=AdnxTQz33y5kwit1v84JwizErq1XqwuCDxD778aUH-QwCwKFInGJ3h36bU8QdgTFIMQ&_nc_zt=24&_nc_ht=scontent.fotp3-3.fna&_nc_gid=oRJx01wii-4dy45Tgx-ryQ&oh=00_AfhOONC6gHhDr0g7o-ddyqw7at6-bHl2iZhb8UWQH_58pA&oe=69250D8E",
- "profile_handle": "facebook",
- "is_sponsored": false,
- "shortcode": "1344226454416038",
- "likes": 12534,
- "post_image": "https://scontent.fotp3-4.fna.fbcdn.net/v/t39.30808-6/583535523_1344102404428443_764020420504838959_n.jpg?stp=dst-jpg_p526x296_tt6&_nc_cat=110&ccb=1-7&_nc_sid=833d8c&_nc_ohc=UwXx_w-yY-4Q7kNvwHI92JR&_nc_oc=AdnIBQD97VrFs4cwsrObV-NB13U0OFu83IukV4n07p9jKd_bGA_GI5OpoufEK8BkeeA&_nc_zt=23&_nc_ht=scontent.fotp3-4.fna&_nc_gid=WHg8XZNKQkBGkIvCnGF3kQ&oh=00_AfgnWX67Rke8m83S2TxFla4c1rJdRxMThFbqBT1O7eGyrg&oe=69250449",
- "post_type": "Post",
- "following": null,
- "link_description_text": null,
- "count_reactions_type": [
- {
- "type": "Like",
- "reaction_count": 9601
- },
- {
- "type": "Love",
- "reaction_count": 2340
- },
- {
- "type": "Care",
- "reaction_count": 353
- },
- {
- "type": "Wow",
- "reaction_count": 119
- },
- {
- "type": "Haha",
- "reaction_count": 92
- },
- {
- "type": "Angry",
- "reaction_count": 21
- },
- {
- "type": "Sad",
- "reaction_count": 8
- }
- ],
- "is_page": true,
- "page_phone": null,
- "page_email": null,
- "page_creation_time": "2007-11-07T00:00:00.000Z",
- "page_reviews_score": null,
- "page_reviewers_amount": null,
- "page_price_range": null,
- "about": [
- {
- "type": "INFLUENCER CATEGORY",
- "value": "Page \u00b7 Internet company",
- "link": null
- },
- {
- "type": "WEBSITE",
- "value": "fb.me/HowToContactFB",
- "link": "https://fb.me/HowToContactFB"
- }
- ],
- "active_ads_urls": [],
- "delegate_page_id": "20531316728",
- "privacy_and_legal_info": null,
- "timestamp": "2025-11-20T16:55:55.934Z",
- "input": {
- "url": "https://www.facebook.com/facebook",
- "num_of_posts": 5,
- "start_date": "",
- "end_date": ""
- }
- }
-]
\ No newline at end of file
diff --git a/tests/samples/instagram/profile.json b/tests/samples/instagram/profile.json
deleted file mode 100644
index 9653911..0000000
--- a/tests/samples/instagram/profile.json
+++ /dev/null
@@ -1,228 +0,0 @@
-{
- "account": "instagram",
- "fbid": "17841400039600391",
- "id": "25025320",
- "followers": 697291572,
- "posts_count": 8241,
- "is_business_account": false,
- "is_professional_account": true,
- "is_verified": true,
- "avg_engagement": 0.0017,
- "external_url": [
- "http://help.instagram.com/"
- ],
- "biography": "Discover what's new on Instagram \ud83d\udd0e\u2728",
- "following": 286,
- "posts": [
- {
- "caption": "painting by mouth \ud83d\udc44\u2063\n \u2063\nVideo by @millybampainti \u2063\nMusic by @opheliawilde.music",
- "comments": 11454,
- "datetime": "2025-11-19T17:17:57.000Z",
- "id": "3769442339278306374",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/581734031_18681801997001321_1932070576932116056_n.jpg?stp=dst-jpg_e15_fr_p1080x1080_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=k_PsIcaWzwwQ7kNvwHXX_2n&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfgmWiSPCR5EYn-4wzkrQ2eEBQ2hUmY8diOXiN9Ou_izxQ&oe=692528A9&_nc_sid=8b3546",
- "likes": 715407,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DRPv9YSADxG",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQO-mxfrthrywUTd_aHwYneykT5hR8alV39J6PyTqACz07xSttT0U4IoE1aG1t2hBkcL4MGqeI7jK7_ni3C0K2lxo3aQxC4NUJT_y9U.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=uidhRAIfHwYQ7kNvwHvlvT9&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6MTIxNTE0ODgwNzE5MjYxMywiYXNzZXRfYWdlX2RheXMiOjAsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjoxOSwidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&vs=f5be72bcf5dcb551&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC8yQzQ0QjIzOTkxN0FCNkQ2RDJCQkFGRTNCMDcyNkI5RF92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HS2RkQVNPM05zclNMUHdDQUJERUdGbnY5d1ZSYnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAmyoCEkPzKqAQVAigCQzMsF0AzXbItDlYEGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&oh=00_Afg8MfrPemi42J4kPLjJ3Jpe7mPzrPnSC1DVvRBU9yQy7g&oe=69213410",
- "is_pinned": false
- },
- {
- "caption": "gliding > walking\n\n#InTheMoment\n\nVideo by @jamalsterrett",
- "comments": 8159,
- "datetime": "2025-11-18T17:05:56.000Z",
- "id": "3768712011689532735",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/582427742_18681652075001321_2703457717514777768_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=52N6A1r_1dkQ7kNvwFBj8R7&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfiU7vFldfFzsRId6VYvtPS3ONiibGG8h7qH8KNDQHEqIg&oe=69251B1F&_nc_sid=8b3546",
- "likes": 690701,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DRNJ5ttgJ0_",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQOa6KfkDlyBaPlGGwha7TzpmnzwLn9HAxE1P3B0ONs62ps2Fa_g65gKg9MDTe8QL0kv5snagf75btalD48NWFpGuEYWvG-Kw0FDiGg.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=-k-i82foR2EQ7kNvwE9t5pI&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6ODYzMzQxMzQyODI0Nzk1LCJhc3NldF9hZ2VfZGF5cyI6MSwidmlfdXNlY2FzZV9pZCI6MTAwOTksImR1cmF0aW9uX3MiOjE1LCJ1cmxnZW5fc291cmNlIjoid3d3In0%3D&ccb=17-1&vs=7242a09d606b124f&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC83MjRCQUJCOUMwNDM4NkMzRjhBMzUyOUI4MDIzNDRBMF92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HQ0FaeENLSE8yUkVHajBFQUNuc20xeWhMeEJfYnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAmtuX4oIrNiAMVAigCQzMsF0AvIcrAgxJvGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&oh=00_AfgPBs1wEI7XT7arXXKYJV5FXv9zGmRh4xQv21XfXIUxWQ&oe=692128BA",
- "is_pinned": false
- },
- {
- "caption": "Fit recap with @mmiriku (Miri) and Ku \ud83d\udd8d\ufe0f\n\nPainting artist and graphic designer Miri created a cartoon character that\u2019s a nod to herself. With short hair and an expressionless face, Ku has become a canvas for showcasing Miri\u2019s weekly outfits. \n\n\u201cFor me, being creative means being free. I\u2019ve always loved fashion and the joy of dressing differently every day. I see outfits as another way to express my art, so this series became a visual diary of that connection.\u201d\n \nVideo by @mmiriku",
- "comments": 4324,
- "datetime": "2025-11-17T20:12:51.000Z",
- "id": "3768080896697163511",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/582240227_18681527452001321_5089760910649723876_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=lf3mHDJM1FIQ7kNvwFwiBJo&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfiIw40HbVqPA_kHrh8AcTqJskj2DLI8UEcehMQuUPP4pA&oe=69250BA9&_nc_sid=8b3546",
- "likes": 255394,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DRK6ZyEkd73",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQMx3Jh8WTOH4HE_MIidqORnBTsMQMX-qFGJEvzrw4JkrIhyBc8yjHrTq7KvWR0hcbR9u7mKq4NNk1FRVBL8UssDb6xRaDiP0R0cZsk.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=GfZQxq-q3U0Q7kNvwGNPFVe&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6MTEyMzI4MDQ4MzEyOTA1OCwiYXNzZXRfYWdlX2RheXMiOjIsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjoyNiwidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&vs=840e4d6031c2976a&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9BOTQ2OTczQTRDOTA0QTUzNURFM0MxNDE3MUE1NjlCOV92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HSlFBd2lLdS03cjAyRUFIQU1LR21qX2l1ZzQ5YnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAmxNvw4sPn_gMVAigCQzMsF0A6XbItDlYEGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&oh=00_Afjn3K-1uGiIWBKdIIa7tbh9kERD1orMq5xugIaJyz5rAQ&oe=69212918",
- "is_pinned": false
- },
- {
- "caption": "Musician @silvanaestradab (Silvana Estrada) finds her roots in family and the timeless sound of her instrument, the cuatro.\u2063\n\u2063\n\u201cWe have to embrace our roots and celebrate and understand that we are in the world because we have so much to give.\u201d \u2063\n\u2063\nHere\u2019s #10Things with Silvana ahead of the @latingrammys (Latin Grammys Awards), where \u201cComo un P\u00e1jaro\u201c was nominated for Best Singer-Songwriter song.\u2063\n\u2063\n1. A moment of silence amid the chaos \ud83e\uddd8\u200d\u2640\ufe0f\u2063\n2. Can we take a second for the fit? \ud83d\udc4f\u2063\n3. When family treasures become good luck charms \ud83e\udd79\u2063\n4. Just a girl and her cuatro \ud83c\udfb6\u2063\n5. A symbol of rebirth \u2728\u2063\n6. Floral on floral \ud83c\udf38\u2063\n7. Music = nostalgia \ud83c\udf0a\u2063\n8. Mirror, mirror on the wall\u2026 \ud83e\udd33\u2063\n9. Celebrating her culture \u2764\ufe0f\u2063\n10. In her element \u2b50",
- "comments": 4316,
- "datetime": "2025-11-17T17:00:50.000Z",
- "id": "3767985117591555557",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/582063811_18681506197001321_6669266777538152909_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=nAoz_5C1IZIQ7kNvwF4o3on&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfiobQSvgGBWe6129K8fA-_u0z2knvdH9PBxjbA0OHCsLw&oe=6925132F&_nc_sid=8b3546",
- "likes": 488444,
- "content_type": "Carousel",
- "url": "https://www.instagram.com/p/DRKkoA1AM3l",
- "video_url": null,
- "is_pinned": false
- },
- {
- "caption": "@vaibhav_sooryavanshi09 (Vaibhav Sooryavanshi) is a cricket legend \u2014 and he\u2019s only 14 years old. \n\nThe all-rounder is the youngest-ever player in the Indian Premier League and is a member of the @rajasthanroyals (Rajasthan Royals). His love for the game started with his dad, who also played cricket and gave Vaibhav his first kit bag at age 5. \n\nSpend a day with Vaibhav at practice, where he shows off his batting and bowling skills and reveals what\u2019s inside his current kit bags. \n\nVaibhav\u2019s advice to other young athletes? \u201cWhatever sport you like, don\u2019t quit playing. If you keep up your hard work, you will get results with time. And you will see your personal improvement in games, too.\u201d",
- "comments": 5958,
- "datetime": "2025-11-16T04:51:37.000Z",
- "id": "3766893314734600553",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/581225632_18681267859001321_7235732305406302514_n.jpg?stp=dst-jpg_e35_p1080x1080_sh0.08_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=LeZzC_sZZZgQ7kNvwFFBNma&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfjbNpoi6JyQCWs6o8knfjASsA0YleVHqGPpnSee1poSTw&oe=69252463&_nc_sid=8b3546",
- "likes": 1071751,
- "content_type": "Carousel",
- "url": "https://www.instagram.com/p/DRGsYMLjLFp",
- "video_url": null,
- "is_pinned": false
- },
- {
- "caption": "pens + desk = insane freestyle \ud83e\udd2f\u2063\n \u2063\n#InTheMoment\u2063\n \u2063\nVideo by @lenstrumental",
- "comments": 26092,
- "datetime": "2025-11-14T17:09:17.000Z",
- "id": "3765814711745052414",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/581257672_1531511241429913_2185789193334358353_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=h0-mzsVmVLIQ7kNvwHkaQEY&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfgRkDglKQ_N5349iRoEvtXNoxvxk6ClqvGleCBE5r_i-Q&oe=69252891&_nc_sid=8b3546",
- "likes": 1725560,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DRC3Ic3gP7-",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQPWYPpLgOef3yX6pCJIRSEdBSafXU4kA4YnaJEUHkNjsCzODjdG7OFmA24sCKwstz81gvkLxEIImtfDt6GGrL5JNLMMhDzlArUrzrs.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=fBs1JsupTZEQ7kNvwGBG8Ap&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6NDQyMzE3NTU2NDU4MzE2NywiYXNzZXRfYWdlX2RheXMiOjUsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjo1NywidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&vs=2428629e2ee008d6&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9DQjREMjc5Q0Q3NDA1OUE2QTU0MzM0RUM2NzgyQURCM192aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HSmZPc0NMaEFSZTR5UlVIQUMxcDl3cEJwV2h3YnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAm_oPzhNq22w8VAigCQzMsF0BM2ZmZmZmaGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&oh=00_AfgajaomMW0pkd9sc3eDw7DLe3rIQBoKOBRHTc5XVrO3tw&oe=69211A9A",
- "is_pinned": false
- },
- {
- "caption": "Her name is Pink and she\u2019s really glad to meet you \ud83c\udfb6\ud83d\udc8b\u2063\n\u2063\nHere\u2019s #10Things from singer @pinkpantheress (PinkPantheress) as she gives us a behind-the-scenes look at her tour in New York, from a fan meet-and-greet to a sold-out show in Brooklyn. \u2063\n\u2063\n1. PinkPantheress is serving looks \ud83d\udd25\u2063\n2. Hair \u2705 Makeup \u2705 Vibes \u2705\u2063\n3. Fan meet-and-greet video inception \ud83c\udfa5\u2063\n4. \u201cPicture in My Mind\u201d \ud83e\udd1d Poster painting\u2063\n5. Costumes for days \u2764\ufe0f\u2063\n6. Working with the same makeup artist >>>\u2063\n7. Did somebody say set list?? \ud83d\udc40\u2063\n8. \ud83c\udfb6 Hey, ooh, is this illegal? \ud83c\udfb6\u2063\n9. Boxes on boxes of doughnuts \ud83d\ude0b\u2063\n10. SOLD OUT!!! \ud83d\udde3\ufe0f",
- "comments": 7969,
- "datetime": "2025-11-13T17:06:48.000Z",
- "id": "3765089019533235772",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/562944142_18680886919001321_3400881731806163989_n.jpg?stp=dst-jpg_e35_p1080x1080_sh0.08_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=VPDlif0yjK0Q7kNvwH9JNRk&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfhZCRsf9PjJK5za4SJJs-hPKKZqQk8-2TBytdbtV2c6zg&oe=692532AE&_nc_sid=8b3546",
- "likes": 622264,
- "content_type": "Carousel",
- "url": "https://www.instagram.com/p/DRASIPVAJY8",
- "video_url": null,
- "is_pinned": false
- },
- {
- "caption": "a wheel is a wheel \ud83e\udd37\n\n#InTheMoment\n\nVideo by @shinverus \nMusic by @teddysphotos",
- "comments": 8264,
- "datetime": "2025-11-12T20:10:36.000Z",
- "id": "3764455947008836411",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/581189459_18680705881001321_5587454374300182126_n.jpg?stp=dst-jpg_e15_fr_p1080x1080_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=0NZpT5FhfAEQ7kNvwFUzrIj&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfgRBeIVGHnLFr6njpX0lP8DAa4FLnjavnbDIwgK32z6hA&oe=69252EF4&_nc_sid=8b3546",
- "likes": 704601,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DQ-CL0mEYM7",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQP8IVMfGMNpzje_guHjee0ajnV5PjlXsD1fa0aM1m_1FM-_hUR4h_j36jFiHcqur6JBnSTBy-1S3jMr-SD8NFWHjE07mxh3rlRk4uQ.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=7jn8srIfdfsQ7kNvwHBoaXg&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6MzExMDQxNjMzMjQ2MDY0OSwiYXNzZXRfYWdlX2RheXMiOjcsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjoxMywidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&vs=95768cd10ffa91a5&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9CRTRFNEM4M0Q1Rjc3QjQyQ0YzODJEQTM5QUJCRkJCNV92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HTFJUcFNKWEU1aDl1Vm9FQUdjVFZtdnNaY0o3YnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAm0snMyYe6hgsVAigCQzMsF0AqAAAAAAAAGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&oh=00_AfhBWuJdi1q_McjiiYd34e6l_VpFBviq2S4NPORneBEG6Q&oe=69210D3C",
- "is_pinned": false
- },
- {
- "caption": "@charles_leclerc (Charles Leclerc) and his pup Leo are racing onto your feed \ud83c\udfce\ufe0f\u2063\n\u2063\nThe Formula 1 driver is back home in Monaco, a place where \u201ctime kind of slows down\u201d and brings back his favorite childhood memories, like hearing the engine noises of the Grand Prix while he was in school.\u2063\n\u2063\nLeo is another spot of joy for Charles. \u201cWhether it\u2019s a good day or a bad day, Leo is always happy and that makes a difference for sure.\u201d \ud83d\udc36\u2063\n\u2063\nPhotos and videos by @antoine",
- "comments": 9444,
- "datetime": "2025-11-12T17:02:25.000Z",
- "id": "3764362036281132587",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/571159454_18680672356001321_6067283357652793275_n.jpg?stp=dst-jpg_e35_p1080x1080_sh0.08_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=EG0kuyUTXaQQ7kNvwEph4ya&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afg7CRaf9YC4lNyMWD_coNqJy_jArf90L8IWn4xKBjNXUw&oe=69251403&_nc_sid=8b3546",
- "likes": 3557269,
- "content_type": "Carousel",
- "url": "https://www.instagram.com/p/DQ9s1PagMYr",
- "video_url": null,
- "is_pinned": false
- },
- {
- "caption": "if you\u2019re seeing this post, it\u2019s your sign to take a moment of zen \ud83e\uddd8\n\nthis waterfall in Brazil is called Cachoeira da Fuma\u00e7a, or \u201cSmoke Falls\u201d \ud83d\ude2e\ud83d\udca8\n\n#InTheMoment\n\nVideo by @marinavieirasou \nMusic by Johann Debussy",
- "comments": 20105,
- "datetime": "2025-11-11T21:09:29.000Z",
- "id": "3763760604772428066",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/580975272_863975589424209_5954144657975698386_n.jpg?stp=dst-jpg_e15_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=K55Nyz9o4AYQ7kNvwGtUDVG&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afi0xQxhYSth2-JlbsUuxFR6yDS9mVKv-wxYRmV7abp98Q&oe=69250A64&_nc_sid=8b3546",
- "likes": 3717926,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DQ7kFQrEeki",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQPEFeiCW6XBves4wKJDUVPj7tkMIkQfclSs49Fh0UUQsrjDtPJj-Ywl0Wk0_ZtuUUsAmu8g6b7bup0uTb__F99GssFlxWQujqqMR9Y.mp4?_nc_cat=1&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=O7I-tMGMR0AQ7kNvwHPWWuX&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6MTQ2MzEzMjc1ODEwMTM2NCwiYXNzZXRfYWdlX2RheXMiOjgsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjo0MCwidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&vs=cd63dcc06f8fa02b&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9CNzQ5QjNFRDA2NzM4MTRDMUVFRDdGNkMyRUUxQTQ4OF92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HTm0yaFNKQjFkUnpIdWxaQUtMZmRPR3ZyUll2YnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAm6LXyxMStmQUVAigCQzMsF0BECHKwIMScGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&oh=00_AfiR5q0MZWJvoUBruxd5zgRoy-zvcyWXmsDx6iWUCg2Oyw&oe=69213AC9",
- "is_pinned": false
- },
- {
- "caption": "Flipping through one of @artbythuraya\u2019s (Thuraya) sketchbooks like\u2026 \u270f\ufe0f\ud83d\udcda \n\nThe artist and graphic designer has been sketching and drawing for as long as she can remember. \u201cI love finding interesting color palettes and I\u2019m always drawn to colorful drawings and designs,\u201d says Thuraya.\n\nHer cure for artist\u2019s block? \u201cI like to paint some pages with neon pink or orange first so it feels less intimidating to draw or paint on them.\u201d \ud83c\udfa8\n \nVideo by @artbythuraya \nMusic by @8salamanda8",
- "comments": 4132,
- "datetime": "2025-11-11T17:08:16.000Z",
- "id": "3763639257256696654",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/574669560_18680273659001321_1858701553672700147_n.jpg?stp=dst-jpg_e15_fr_p1080x1080_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=0LVdCDuRBp4Q7kNvwGkIu4w&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfjWpj0PsyTFXb6J98IdULJjluXkJ8gaFz_2YZhI_vU6Mw&oe=69253403&_nc_sid=8b3546",
- "likes": 305332,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DQ7Ifa_gBtO",
- "video_url": "https://scontent-fra3-2.cdninstagram.com/o1/v/t2/f2/m86/AQPJ5m9jYNnVN2_xKT8iKe1InFL-S2TQF5gqn9H9wncP2xnTwvs3Cg41QhXRm7jFOafn0W6A5QzvDN75IYlmXoRpT15P7FWRdfC5JV4.mp4?_nc_cat=111&_nc_sid=5e9851&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_ohc=Jnm96UiO86cQ7kNvwFHkKJG&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6MTM4NzcyNzY2NjIwMzYzNCwiYXNzZXRfYWdlX2RheXMiOjgsInZpX3VzZWNhc2VfaWQiOjEwMDk5LCJkdXJhdGlvbl9zIjo2LCJ1cmxnZW5fc291cmNlIjoid3d3In0%3D&ccb=17-1&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&vs=434223c562bfcfd4&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC9GQjRCREY0QjcyRkRCNzBCMzkwMDU5N0Q2NjEzQkZBRV92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HTmpfa1NMelJWLWsyeVlFQUFtRDhHd1FLejVvYnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAm5K-26bCI9wQVAigCQzMsF0AYqfvnbItEGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&oh=00_AfgpVH9AzIPOseNRW-ZSvc0hyEs2zbaNZD9YFS0piiApug&oe=69213EB4",
- "is_pinned": false
- },
- {
- "caption": "@ariana_greenblatt\u2019s (Ariana Greenblatt) camera roll is pure magic \ud83e\ude84\u2728\u2063\n \u2063\nIn today\u2019s episode of #WhatsInMyCameraRoll, the actress shows off photos from:\u2063\n \u2063\n\n\ud83e\uddc0 a three-hour hunt for mac and cheese with @dominic.sessa (Dominic Sessa)\u2063\n\ud83e\udee3 stunt work gone wrong\u2063\n\ud83c\udfa5 never-before-seen BTS of her new movie @nysmmovie (\u201cNow You See Me: Now You Don\u2019t\u201d)",
- "comments": 4969,
- "datetime": "2025-11-10T20:03:10.000Z",
- "id": "3763002910675382648",
- "image_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-15/580702200_18679965823001321_2764781517024588673_n.jpg?stp=dst-jpg_e35_p1080x1080_sh0.08_tt6&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=B_myOTv3LEcQ7kNvwFk_Mw_&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afh4ZryUf08EQoG5a4BZYX8MsOmNV1po_eGIcQ487-JMcA&oe=69251893&_nc_sid=8b3546",
- "likes": 321015,
- "content_type": "Video",
- "url": "https://www.instagram.com/p/DQ43zXDkTF4",
- "video_url": "https://scontent-fra5-2.cdninstagram.com/o1/v/t2/f2/m86/AQPF9xQpIA1Lx413WZGH6TTatp3DDVZe4tzaKn4Ijcw_ZttODA7zLD8ULhNlA-vHSw6q4WTsBzqcfsUz4auU0iSr8DUT3SPg3fvC5n8.mp4?_nc_cat=109&_nc_sid=5e9851&_nc_ht=scontent-fra5-2.cdninstagram.com&_nc_ohc=jEJV1ukrYqMQ7kNvwE6dMre&efg=eyJ2ZW5jb2RlX3RhZyI6Inhwdl9wcm9ncmVzc2l2ZS5JTlNUQUdSQU0uQ0xJUFMuQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSIsInhwdl9hc3NldF9pZCI6ODc2NzMyMDE4MzYxMDM5LCJhc3NldF9hZ2VfZGF5cyI6OSwidmlfdXNlY2FzZV9pZCI6MTAwOTksImR1cmF0aW9uX3MiOjE3MCwidXJsZ2VuX3NvdXJjZSI6Ind3dyJ9&ccb=17-1&vs=17f0d6dfa828a48f&_nc_vs=HBksFQIYUmlnX3hwdl9yZWVsc19wZXJtYW5lbnRfc3JfcHJvZC82NjRBRTdGOUE0MEJFNTIyQTdGMkYyQzJBNkI1N0NCNl92aWRlb19kYXNoaW5pdC5tcDQVAALIARIAFQIYOnBhc3N0aHJvdWdoX2V2ZXJzdG9yZS9HSDVha2lJXy1RRjh3endIQUlFOEh1VHFlbUpSYnN0VEFRQUYVAgLIARIAKAAYABsCiAd1c2Vfb2lsATEScHJvZ3Jlc3NpdmVfcmVjaXBlATEVAAAmnoukyMLYjgMVAigCQzMsF0BlQQ5WBBiTGBJkYXNoX2Jhc2VsaW5lXzFfdjERAHX-B2XmnQEA&_nc_gid=cw6-_j-JhTMw7N2bbykfug&_nc_zt=28&oh=00_AfgKOkWv2hBTWKD8iGRU7nTVYNimoKAKA1iM-Hd_sp8fFw&oe=69212F1E",
- "is_pinned": false
- }
- ],
- "profile_image_link": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/550891366_18667771684001321_1383210656577177067_n.jpg?stp=dst-jpg_s320x320_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=yJDuf_37I78Q7kNvwFwPPhF&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfiiZ25Szwb6Ps1PZVYRkQhp_UuzD1XQ5IB2relEmPEM2w&oe=69251AF1&_nc_sid=8b3546",
- "profile_url": "https://instagram.com/instagram",
- "profile_name": "Instagram",
- "highlights_count": 15,
- "full_name": "Instagram",
- "is_private": false,
- "url": "https://www.instagram.com/instagram",
- "is_joined_recently": false,
- "has_channel": false,
- "partner_id": "25025320",
- "business_address": null,
- "related_accounts": [
- {
- "id": "47913961291",
- "profile_name": "\uc870\uc720\ub9ac JO YURI",
- "is_private": false,
- "is_verified": true,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/448149897_318348131333718_5639948001191412494_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby40OTcuYzIifQ&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=UrsCtrnb1W4Q7kNvwGSAzZ7&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfiBxEJ-HC3K_7Ec1qYH2P7vDhpeGwcFdBFUTdgRx6_f4w&oe=692517A5&_nc_sid=8b3546",
- "user_name": "zo__glasss"
- },
- {
- "id": "52057517181",
- "profile_name": "\u8a2d\u5b9a\u305b\u3076\u3093",
- "is_private": false,
- "is_verified": false,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/329419233_145796804994270_5889321886093160950_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=wnUOQU9uh2UQ7kNvwEHIFeZ&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afh4oZ06-S8RWGQZlPWSMs41jBbXp7G3utpz8L72ApZXYw&oe=69251676&_nc_sid=8b3546",
- "user_name": "settei.seven"
- },
- {
- "id": "61519339885",
- "profile_name": "ILLIT \uc544\uc77c\ub9bf",
- "is_private": false,
- "is_verified": true,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/571115836_17951810346051886_1465137572491758307_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby40OTkuYzIifQ&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=mLZMFzfMwYYQ7kNvwEgmfTe&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afje0N18lXuD49fwq0rvEs-JGaAvMt0ri6CLrNm7zcuPYw&oe=692518A9&_nc_sid=8b3546",
- "user_name": "illit_official"
- },
- {
- "id": "61944716934",
- "profile_name": "TWS (\ud22c\uc5b4\uc2a4)",
- "is_private": false,
- "is_verified": true,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/560548764_17943106626068935_7992087485001898401_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby43NTAuYzIifQ&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=yNI2quk4ALwQ7kNvwFSuXyM&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afj3-f6IUVUlNYKKuHB841POvSnMR8vUbYj6S2LWfztcnQ&oe=69250311&_nc_sid=8b3546",
- "user_name": "tws_pledis"
- },
- {
- "id": "11927071408",
- "profile_name": "\u110b\u1175\u11b7\u1109\u1175\u110b\u116a\u11ab",
- "is_private": false,
- "is_verified": true,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/470924210_631456425886325_6886504717911321733_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDUxLmMyIn0&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=KKfPRFDBSJgQ7kNvwGE_Fa5&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_Afjmtxy_Cq7lhOa-YMVAY-37jRLA47gadQmQcbi7UI1C_A&oe=692531D0&_nc_sid=8b3546",
- "user_name": "yim_siwang"
- },
- {
- "id": "67066633135",
- "profile_name": "Atrass\u3010\u30a2\u30c8\u30e9\u30b9\u3011",
- "is_private": false,
- "is_verified": false,
- "profile_pic_url": "https://scontent-fra3-2.cdninstagram.com/v/t51.2885-19/447197239_473707615114615_6794268554276293899_n.jpg?stp=dst-jpg_s150x150_tt6&efg=eyJ2ZW5jb2RlX3RhZyI6InByb2ZpbGVfcGljLmRqYW5nby4xMDgwLmMyIn0&_nc_ht=scontent-fra3-2.cdninstagram.com&_nc_cat=1&_nc_oc=Q6cZ2QHRNKnpTyp3nOTa90wqCPSVZpi-KuApYBSwHsZkqNswtqlwIfFChTfLlBJQSDbpzdg&_nc_ohc=XsKyIhMs29cQ7kNvwH8zBNX&_nc_gid=cw6-_j-JhTMw7N2bbykfug&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfjJ7BiSgOcx4eZlTnCWgF5VA5_JlZrbyeeBTUtHunFJOA&oe=692530BA&_nc_sid=8b3546",
- "user_name": "atrass_wingashan"
- }
- ],
- "email_address": null,
- "timestamp": "2025-11-20T16:54:51.664Z",
- "input": {
- "url": "https://www.instagram.com/instagram"
- }
-}
\ No newline at end of file
diff --git a/tests/samples/linkedin/profile.json b/tests/samples/linkedin/profile.json
deleted file mode 100644
index ed81411..0000000
--- a/tests/samples/linkedin/profile.json
+++ /dev/null
@@ -1,407 +0,0 @@
-{
- "id": "williamhgates",
- "name": "Bill Gates",
- "city": "Seattle, Washington, United States",
- "country_code": "US",
- "position": "Chair, Gates Foundation and Founder, Breakthrough Energy",
- "about": "Chair of the Gates Foundation. Founder of Breakthrough Energy. Co-founder of Microsoft. Voracious reader. Avid traveler. Active blogger.",
- "posts": [
- {
- "title": "Saving lives, cutting emissions, and staying resilient in a warming world",
- "attribution": "I recently published a long essay about climate change on the Gates Notes. This is the first of four newsletters I\u2019ll\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/saving-lives-cutting-emissions-staying-resilient-warming-bill-gates-jstyc",
- "created_at": "2025-10-29T00:00:00.000Z",
- "interaction": "5,43 - 989 Comments",
- "id": "7389128335357947904"
- },
- {
- "title": "We\u2019re closer than ever to eradicating polio",
- "attribution": "..",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/were-closer-than-ever-eradicating-polio-bill-gates-wyhac",
- "created_at": "2025-10-18T00:00:00.000Z",
- "interaction": "5,81 - 719 Comments",
- "id": "7385166929856172032"
- },
- {
- "title": "Demystifying the science behind fission and fusion",
- "attribution": "I\u2019m lucky to learn firsthand about some of the world\u2019s most cutting-edge technologies. I\u2019ve seen artificial\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/demystifying-science-behind-fission-fusion-bill-gates-ylhic",
- "created_at": "2025-10-11T00:00:00.000Z",
- "interaction": "5,39 - 727 Comments",
- "id": "7382558042824855552"
- },
- {
- "title": "Utah\u2019s hottest new power source is 15,000 feet below the ground",
- "attribution": "When my son, Rory, was younger, we used to love visiting power plants together. It was the perfect father-son activity\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/utahs-hottest-new-power-source-15000-feet-below-ground-bill-gates-otlwc",
- "created_at": "2025-09-30T00:00:00.000Z",
- "interaction": "5,99 - 661 Comments",
- "id": "7378858122087616513"
- },
- {
- "title": "Why I\u2019m Still Optimistic About Global Health",
- "attribution": "I recently wrote this essay for TIME Magazine about why I'm still optimistic about global health: One of humanity\u2019s\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/why-im-still-optimistic-global-health-bill-gates-ji9xc",
- "created_at": "2025-09-23T00:00:00.000Z",
- "interaction": "4,33 - 765 Comments",
- "id": "7376347643272343554"
- },
- {
- "title": "This is how a parasite helped build the CDC and changed public health forever",
- "attribution": "I spend a lot of time thinking and worrying about malaria. After all, it\u2019s one of the big focuses of my work at the\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/how-parasite-helped-build-cdc-changed-public-health-forever-gates-xvhlc",
- "created_at": "2025-08-26T00:00:00.000Z",
- "interaction": "4,10 - 613 Comments",
- "id": "7366207310018375680"
- },
- {
- "title": "One of the most unique and supportive learning environments I have ever heard of",
- "attribution": "When I was a kid, I couldn\u2019t sit still. My teachers used to get mad at me for squirming in my chair and chewing on my\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/one-most-unique-supportive-learning-environments-i-have-bill-gates-e3fcc",
- "created_at": "2025-08-13T00:00:00.000Z",
- "interaction": "5,59 - 901 Comments",
- "id": "7361457006081134592"
- },
- {
- "title": "This heroic nurse climbs 1000-foot ladders to save lives",
- "attribution": "How do you get to work? Some people roll out of bed and move 10 feet to their desk. Others walk to the office or take\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/heroic-nurse-climbs-1000-foot-ladders-save-lives-bill-gates-gh0ic",
- "created_at": "2025-07-31T00:00:00.000Z",
- "interaction": "5,85 - 823 Comments",
- "id": "7356808124818735104"
- },
- {
- "title": "A gut-wrenching problem we can solve",
- "attribution": "In 1997, I came across a New York Times column by Nick Kristof that stopped me in my tracks. The headline was \u201cFor\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/gut-wrenching-problem-we-can-solve-bill-gates-ahczc",
- "created_at": "2025-07-27T00:00:00.000Z",
- "interaction": "6,27 - 1,154 Comments",
- "id": "7354909425704292352"
- },
- {
- "title": "A book about tuberculosis, and everything else",
- "attribution": "What do Adirondack chairs, Stetson hats, the city of Pasadena, and World War I have in common? According to John Green,\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5612AQHn1kRpjWsY7A/article-cover_image-shrink_720_1280/B56Zot1D3DG0AM-/0/1761705477503?e=2147483647&v=beta&t=wnrHrl7BgWHpsXAXFEOUbcmFE0tnxibZ-Ze5ESzbKMs",
- "link": "https://www.linkedin.com/pulse/book-tuberculosis-everything-else-bill-gates-5ibhc",
- "created_at": "2025-07-24T00:00:00.000Z",
- "interaction": "4,40 - 668 Comments",
- "id": "7354250885624946688"
- }
- ],
- "current_company": {
- "name": "Gates Foundation",
- "company_id": "gates-foundation",
- "title": "Co-chair",
- "location": null
- },
- "experience": [
- {
- "title": "Co-chair",
- "description_html": null,
- "start_date": "2000",
- "end_date": "Present",
- "company": "Gates Foundation",
- "company_id": "gates-foundation",
- "url": "https://www.linkedin.com/company/gates-foundation",
- "company_logo_url": "https://media.licdn.com/dms/image/v2/D560BAQEgMqqFTd40Tg/company-logo_100_100/company-logo_100_100/0/1736784969376/bill__melinda_gates_foundation_logo?e=2147483647&v=beta&t=2JH2cMcZms60vPAMbvVZyMeYXosQ1Jjy5axDlyeQ1Ww"
- },
- {
- "title": "Founder",
- "description_html": null,
- "start_date": "2015",
- "end_date": "Present",
- "company": "Breakthrough Energy",
- "company_id": "breakthrough-energy",
- "url": "https://www.linkedin.com/company/breakthrough-energy",
- "company_logo_url": "https://media.licdn.com/dms/image/v2/D560BAQFRMYiQN7-2kA/company-logo_100_100/B56ZoI4SGPI0AQ-/0/1761085563539/breakthrough_energy_logo?e=2147483647&v=beta&t=J6RbEvs17fl1uiEaXQm0hmXy4imx36mV_Hu80JcR1DE"
- },
- {
- "title": "Co-founder",
- "description_html": null,
- "start_date": "1975",
- "end_date": "Present",
- "company": "Microsoft",
- "company_id": "microsoft",
- "url": "https://www.linkedin.com/company/microsoft",
- "company_logo_url": "https://media.licdn.com/dms/image/v2/D560BAQH32RJQCl3dDQ/company-logo_100_100/B56ZYQ0mrGGoAU-/0/1744038948046/microsoft_logo?e=2147483647&v=beta&t=rr_7_bFRKp6umQxIHErPOZHtR8dMPIYeTjlKFdotJBY"
- }
- ],
- "url": "https://tr.linkedin.com/in/williamhgates",
- "people_also_viewed": [
- {
- "profile_link": "https://www.linkedin.com/in/melindagates",
- "name": "Melinda French Gates",
- "about": null,
- "location": "United States"
- },
- {
- "profile_link": "https://www.linkedin.com/in/tyleralterman",
- "name": "Tyler Alterman",
- "about": null,
- "location": "Brooklyn, NY"
- },
- {
- "profile_link": "https://www.linkedin.com/in/toddjduckett",
- "name": "Todd J. Duckett",
- "about": null,
- "location": "Lansing, MI"
- },
- {
- "profile_link": "https://is.linkedin.com/in/hallatomasdottir",
- "name": "Halla Tomasdottir",
- "about": null,
- "location": "Iceland"
- },
- {
- "profile_link": "https://www.linkedin.com/in/matthew-swift-8ba7529",
- "name": "Matthew Swift",
- "about": null,
- "location": "Palm Beach, FL"
- },
- {
- "profile_link": "https://www.linkedin.com/in/petefishman",
- "name": "Peter Fishman",
- "about": null,
- "location": "San Francisco, CA"
- },
- {
- "profile_link": "https://www.linkedin.com/in/sherryb",
- "name": "\u2726 Sherry Whitaker Budziak",
- "about": null,
- "location": "Deerfield, IL"
- },
- {
- "profile_link": "https://www.linkedin.com/in/tonyteravainen",
- "name": "Tony Teravainen PMP CSSBB",
- "about": null,
- "location": "San Diego, CA"
- },
- {
- "profile_link": "https://www.linkedin.com/in/charlesmarohn",
- "name": "Charles Marohn",
- "about": null,
- "location": "Brainerd, MN"
- },
- {
- "profile_link": "https://www.linkedin.com/in/schm1tt",
- "name": "Patrick Schmitt",
- "about": null,
- "location": "New York, NY"
- },
- {
- "profile_link": "https://www.linkedin.com/in/melindalackey",
- "name": "Melinda Lackey",
- "about": null,
- "location": "New York, NY"
- },
- {
- "profile_link": "https://www.linkedin.com/in/bill-cronin-5490492",
- "name": "Bill Cronin",
- "about": null,
- "location": "Odessa, FL"
- },
- {
- "profile_link": "https://www.linkedin.com/in/ezohn",
- "name": "Ethan Zohn",
- "about": null,
- "location": "Hillsborough County, NH"
- },
- {
- "profile_link": "https://www.linkedin.com/in/gary-taubes-942a6459",
- "name": "Gary Taubes",
- "about": null,
- "location": "Oakland, CA"
- },
- {
- "profile_link": "https://www.linkedin.com/in/sharonhenifin",
- "name": "Sharon Henifin, CLC, CN-BA",
- "about": null,
- "location": "Portland, Oregon Metropolitan Area"
- },
- {
- "profile_link": "https://www.linkedin.com/in/josephrrusso",
- "name": "Joseph Russo",
- "about": null,
- "location": "West Palm Beach, FL"
- },
- {
- "profile_link": "https://www.linkedin.com/in/jasongrad",
- "name": "Jason Grad",
- "about": null,
- "location": "New York, NY"
- },
- {
- "profile_link": "https://www.linkedin.com/in/mrdaikensjr",
- "name": "Dwayne Aikens Jr.",
- "about": null,
- "location": "Oakland, CA"
- },
- {
- "profile_link": "https://www.linkedin.com/in/erikrees",
- "name": "Erik Rees",
- "about": null,
- "location": "Rancho Santa Margarita, CA"
- }
- ],
- "educations_details": "Harvard University",
- "education": [
- {
- "title": "Harvard University",
- "url": "https://www.linkedin.com/school/harvard-university/?trk=public_profile_school_profile-section-card_image-click",
- "start_year": "1973",
- "end_year": "1975",
- "description": null,
- "description_html": null,
- "institute_logo_url": "https://media.licdn.com/dms/image/v2/C4E0BAQF5t62bcL0e9g/company-logo_100_100/company-logo_100_100/0/1631318058235?e=2147483647&v=beta&t=Ye1klXowyo8TIcnkhTlmORgiA5ZywvooNihDMnx5urQ"
- },
- {
- "title": "Lakeside School",
- "url": "https://www.linkedin.com/school/lakeside-school/?trk=public_profile_school_profile-section-card_image-click",
- "description": null,
- "description_html": null,
- "institute_logo_url": "https://media.licdn.com/dms/image/v2/D560BAQGFmOQmzpxg9A/company-logo_100_100/company-logo_100_100/0/1683732883164/lakeside_school_logo?e=2147483647&v=beta&t=EmadOLH7MckKZvCCrgmAOikCRtzVRtqqN4PJi35CNyo"
- }
- ],
- "avatar": "https://media.licdn.com/dms/image/v2/D5603AQF-RYZP55jmXA/profile-displayphoto-shrink_200_200/B56ZRi8g.aGsAY-/0/1736826818802?e=2147483647&v=beta&t=bKWfN6UwwtiCqFWsG7rBELbd48qJOAMLdxhBzzkJV0k",
- "followers": 39312887,
- "connections": 8,
- "current_company_company_id": "gates-foundation",
- "current_company_name": "Gates Foundation",
- "location": "Seattle",
- "input_url": "https://www.linkedin.com/in/williamhgates",
- "linkedin_id": "williamhgates",
- "activity": [
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_luxwall-a-breakthrough-energybacked-company-activity-7397039090300289024-i8M3",
- "title": "LuxWall, a Breakthrough Energy\u2013backed company, is growing in Detroit\u2014and bringing new jobs along with it.",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7397039090300289024"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_five-years-ago-just-two-months-after-my-activity-7396302164459102208-Kbj8",
- "title": "Five years ago, just two months after my dad died from Alzheimer's disease, I worked with a coalition of partners to create the Alzheimer's Disease\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5622AQHgEdBt8av3CQ/feedshare-shrink_800/B56ZqTxoD2JYAg-/0/1763415851860?e=2147483647&v=beta&t=zCTCb6zxupuvG6lfR8wLNsSR3EqB6U_q8wRVDtTI0uY",
- "id": "7396302164459102208"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_fighting-climate-change-requires-actions-activity-7393808373814685696-r4Ed",
- "title": "Fighting climate change requires actions on two fronts: cutting emissions and protecting vulnerable people. I will continue to invest billions in\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5622AQHHcm91usLudw/feedshare-shrink_2048_1536/B56ZpwViXrJQAw-/0/1762821285730?e=2147483647&v=beta&t=xBCPzIccwCP53aFG20U2hyamr2xJphdmDENxCqeTQoc",
- "id": "7393808373814685696"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_my-commitment-to-fightingand-solvingclimate-activity-7393404126505689088-fiqd",
- "title": "My commitment to fighting\u2014and solving\u2014climate change has not wavered. In addition to the billions I am investing in innovation that will help the\u2026",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7393404126505689088"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_sa-becomes-the-first-african-country-to-register-activity-7393120672111185920-SKf2",
- "title": "South Africa\u2019s Lenacapavir rollout is a signal that progress is possible when innovation meets urgency.",
- "img": "https://media.licdn.com/dms/image/sync/v2/D4D27AQFNkSDu_tpZ7g/articleshare-shrink_1280_800/B4DZokVReGJIAQ-/0/1761596917780?e=2147483647&v=beta&t=2XH0BTMGQJgud_VJq-Oyfz5VVFcOjzPQmlKWvkiI0GQ",
- "id": "7393120672111185920"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_to-strengthen-human-welfare-globally-we-activity-7392748129042907137-9s6_",
- "title": "To strengthen human welfare globally, we must help the most vulnerable communities adapt to a warming planet while continuing to invest in critical\u2026",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7392748129042907137"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_when-i-started-breakthrough-energy-the-world-activity-7392049630605099008-WCuo",
- "title": "When I started Breakthrough Energy, the world needed affordable clean energy solutions that didn\u2019t exist yet. \u200b \u200b Affordable, reliable, clean energy\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D4D05AQGF8BR-A7TzTw/videocover-high/B4DZpXV5dsG8BU-/0/1762401954068?e=2147483647&v=beta&t=p-h5YEqqlB4cWDe0JicwMiFaNOi_iHMZSdG3L6PGjzo",
- "id": "7392049630605099008"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_today-i-visited-the-alzheimers-therapeutic-activity-7391650830199775233-cR81",
- "title": "Today I visited the Alzheimer's Therapeutic Research Institute (ATRI) at USC, led by Dr. Paul Aisen, to learn more about the current landscape of\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D5622AQEFYSq5diGoKg/feedshare-shrink_800/B56ZpRrQg3HQAk-/0/1762306886686?e=2147483647&v=beta&t=5dpkTj8DyD87d5jbw-ylidifhkJdkVtMwdfKVu0l3cQ",
- "id": "7391650830199775233"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_were-on-the-brink-of-eradicating-polio-for-activity-7391179821646716928-EGiI",
- "title": "We\u2019re on the brink of eradicating polio for good. It would be a deadly mistake to back down from the fight now.",
- "img": "https://media.licdn.com/dms/image/v2/D5605AQFMop8kHgEtkQ/videocover-high/B56ZpK.wx4HYBU-/0/1762194575348?e=2147483647&v=beta&t=LQJGu7ZZLYlFrnGcfHpwAhnl0K_bIn94RQT4_hZh5Z0",
- "id": "7391179821646716928"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_this-is-an-exciting-partnership-with-alzheimers-activity-7390903402932813824-mTWv",
- "title": "This is an exciting partnership with Alzheimer's Research UK. Answering these questions could change the course of our fight against Alzheimer\u2019s.",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7390903402932813824"
- },
- {
- "interaction": "Liked by Bill Gates",
- "link": "https://www.linkedin.com/posts/alzheimer%27s-research-uk_today-marks-a-pivotal-moment-in-the-global-activity-7387141173267816448-bf5I",
- "title": "Today marks a pivotal moment in the global fight against dementia. Alzheimer\u2019s Research UK, alongside Gates Ventures are proud to launch a\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D4E10AQELRUrvrLBxFQ/ads-video-thumbnail_720_1280/B4EZoRlo13KsAc-/0/1761231670956?e=2147483647&v=beta&t=skARaYStlXOrE0cNE5CgdPYQx4cELDW8kdRu6XuacsI",
- "id": "7387141173267816448"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_im-grateful-for-people-like-john-and-nancy-activity-7390459116651155457-2aYl",
- "title": "I\u2019m grateful for people like John and Nancy from Rotary International\u2014leaders whose courage and commitment bring us closer to a polio-free world\u2026",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7390459116651155457"
- },
- {
- "interaction": "Liked by Bill Gates",
- "link": "https://www.linkedin.com/posts/nancy-barbee-18a6308_i-sat-next-to-bill-gates-at-the-gates-foundation-activity-7388529939463180288-lJiu",
- "title": "I sat next to Bill Gates at the Gates Foundation media event for World Polio Day 2025 Bill is the person who inspired me to start leading Rotarians\u2026",
- "img": "https://media.licdn.com/dms/image/v2/D4E22AQFXY_wl5a3-Hg/feedshare-shrink_800/B4EZokJLI7GYAg-/0/1761542977553?e=2147483647&v=beta&t=T9wB3225_8CIbeVs6KZae4GRuM3jJHNAdZCXsHm76Hk",
- "id": "7388529939463180288"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_a-new-approach-for-the-worlds-climate-strategy-activity-7390110248264466432-tub6",
- "title": "Climate change is one of the most pressing challenges the world faces today. The good news is that we've made incredible progress in recent years\u2026",
- "img": "https://media.licdn.com/dms/image/sync/v2/D4E27AQEwAcMGPj_kKA/articleshare-shrink_1280_800/B4EZopJuU0KoAQ-/0/1761627006826?e=2147483647&v=beta&t=dx6lfbhpJMJ2nd-K-gmIFrS_odoBhduCEfYVgUhGolY",
- "id": "7390110248264466432"
- },
- {
- "interaction": "Shared by Bill Gates",
- "link": "https://www.linkedin.com/posts/williamhgates_congratulations-on-this-well-deserved-award-activity-7388262728068452352-N6SR",
- "title": "Congratulations on this well-deserved award. I\u2019m grateful for your leadership and commitment to ensuring everyone can live a healthy, prosperous life.",
- "img": "https://static.licdn.com/aero-v1/sc/h/53n89ecoxpr1qrki1do3alazb",
- "id": "7388262728068452352"
- }
- ],
- "linkedin_num_id": "251749025",
- "banner_image": "https://media.licdn.com/dms/image/v2/D5616AQEjhPbTCeblYg/profile-displaybackgroundimage-shrink_200_800/B56ZcytR5SGsAc-/0/1748902420393?e=2147483647&v=beta&t=a-tBeZkxzWTHWYY6MAjxt0oTEuxlW33EUkK3gm5_te4",
- "honors_and_awards": null,
- "similar_profiles": [],
- "default_avatar": false,
- "memorialized_account": false,
- "bio_links": [
- {
- "title": "Blog",
- "link": "https://gatesnot.es/sourcecode-li"
- }
- ],
- "first_name": "Bill",
- "last_name": "Gates",
- "timestamp": "2025-11-20T17:04:28.062Z",
- "input": {
- "url": "https://www.linkedin.com/in/williamhgates"
- }
-}
\ No newline at end of file
diff --git a/tests/samples/serp/google.json b/tests/samples/serp/google.json
deleted file mode 100644
index a6727ca..0000000
--- a/tests/samples/serp/google.json
+++ /dev/null
@@ -1,23 +0,0 @@
-[
- {
- "position": 1,
- "title": "Pizza Hut | Delivery & Carryout - No One OutPizzas The Hut!",
- "url": "https://www.pizzahut.com/",
- "description": "Discover classic & new menu items, find deals and enjoy seamless ordering for delivery and carryout. No One OutPizzas the Hut\u00ae.",
- "displayed_url": "https://www.pizzahut.com"
- },
- {
- "position": 2,
- "title": "Pizza",
- "url": "https://en.wikipedia.org/wiki/Pizza",
- "description": "Pizza is an Italian dish typically consisting of a flat base of leavened wheat-based dough topped with tomato, cheese, and other ingredients, baked at a ...",
- "displayed_url": "https://en.wikipedia.org \u203a wiki \u203a Pizza"
- },
- {
- "position": 3,
- "title": "Domino's: Pizza Delivery & Carryout, Pasta, Wings & More",
- "url": "https://www.dominos.com/",
- "description": "PRICES HIGHER FOR SOME LOCATIONS. Treat yo self to our best, most premium medium Specialty Pizzas for just $9.99 each when you Mix & Match.",
- "displayed_url": "https://www.dominos.com"
- }
-]
\ No newline at end of file
diff --git a/tests/samples/web_unlocker/country_targeting.html b/tests/samples/web_unlocker/country_targeting.html
deleted file mode 100644
index c07a7cf..0000000
--- a/tests/samples/web_unlocker/country_targeting.html
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "headers": {
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
- "Accept-Encoding": "gzip, deflate, br, zstd",
- "Accept-Language": "en-US,en;q=0.9",
- "Host": "httpbin.org",
- "Sec-Ch-Ua": "\"Chromium\";v=\"142\", \"Microsoft Edge\";v=\"142\", \"Not_A Brand\";v=\"99\"",
- "Sec-Ch-Ua-Platform": "\"Windows\"",
- "Sec-Fetch-Dest": "empty",
- "Sec-Fetch-Mode": "cors",
- "Sec-Fetch-Site": "none",
- "Sec-Fetch-User": "?0",
- "Upgrade-Insecure-Requests": "1",
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0",
- "X-Amzn-Trace-Id": "Root=1-691f5229-7d5c92055198bdba39341a7f"
- }
-}
diff --git a/tests/samples/web_unlocker/multiple_urls_1.html b/tests/samples/web_unlocker/multiple_urls_1.html
deleted file mode 100644
index d55209d..0000000
--- a/tests/samples/web_unlocker/multiple_urls_1.html
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-
-
-
Herman Melville - Moby-Dick
-
-
-
- Availing himself of the mild, summer-cool weather that now reigned in these latitudes, and in preparation for the peculiarly active pursuits shortly to be anticipated, Perth, the begrimed, blistered old blacksmith, had not removed his portable forge to the hold again, after concluding his contributory work for Ahab's leg, but still retained it on deck, fast lashed to ringbolts by the foremast; being now almost incessantly invoked by the headsmen, and harpooneers, and bowsmen to do some little job for them; altering, or repairing, or new shaping their various weapons and boat furniture. Often he would be surrounded by an eager circle, all waiting to be served; holding boat-spades, pike-heads, harpoons, and lances, and jealously watching his every sooty movement, as he toiled. Nevertheless, this old man's was a patient hammer wielded by a patient arm. No murmur, no impatience, no petulance did come from him. Silent, slow, and solemn; bowing over still further his chronically broken back, he toiled away, as if toil were life itself, and the heavy beating of his hammer the heavy beating of his heart. And so it was.—Most miserable! A peculiar walk in this old man, a certain slight but painful appearing yawing in his gait, had at an early period of the voyage excited the curiosity of the mariners. And to the importunity of their persisted questionings he had finally given in; and so it came to pass that every one now knew the shameful story of his wretched fate. Belated, and not innocently, one bitter winter's midnight, on the road running between two country towns, the blacksmith half-stupidly felt the deadly numbness stealing over him, and sought refuge in a leaning, dilapidated barn. The issue was, the loss of the extremities of both feet. Out of this revelation, part by part, at last came out the four acts of the gladness, and the one long, and as yet uncatastrophied fifth act of the grief of his life's drama. He was an old man, who, at the age of nearly sixty, had postponedly encountered that thing in sorrow's technicals called ruin. He had been an artisan of famed excellence, and with plenty to do; owned a house and garden; embraced a youthful, daughter-like, loving wife, and three blithe, ruddy children; every Sunday went to a cheerful-looking church, planted in a grove. But one night, under cover of darkness, and further concealed in a most cunning disguisement, a desperate burglar slid into his happy home, and robbed them all of everything. And darker yet to tell, the blacksmith himself did ignorantly conduct this burglar into his family's heart. It was the Bottle Conjuror! Upon the opening of that fatal cork, forth flew the fiend, and shrivelled up his home. Now, for prudent, most wise, and economic reasons, the blacksmith's shop was in the basement of his dwelling, but with a separate entrance to it; so that always had the young and loving healthy wife listened with no unhappy nervousness, but with vigorous pleasure, to the stout ringing of her young-armed old husband's hammer; whose reverberations, muffled by passing through the floors and walls, came up to her, not unsweetly, in her nursery; and so, to stout Labor's iron lullaby, the blacksmith's infants were rocked to slumber. Oh, woe on woe! Oh, Death, why canst thou not sometimes be timely? Hadst thou taken this old blacksmith to thyself ere his full ruin came upon him, then had the young widow had a delicious grief, and her orphans a truly venerable, legendary sire to dream of in their after years; and all of them a care-killing competency.
-
- Availing himself of the mild, summer-cool weather that now reigned in these latitudes, and in preparation for the peculiarly active pursuits shortly to be anticipated, Perth, the begrimed, blistered old blacksmith, had not removed his portable forge to the hold again, after concluding his contributory work for Ahab's leg, but still retained it on deck, fast lashed to ringbolts by the foremast; being now almost incessantly invoked by the headsmen, and harpooneers, and bowsmen to do some little job for them; altering, or repairing, or new shaping their various weapons and boat furniture. Often he would be surrounded by an eager circle, all waiting to be served; holding boat-spades, pike-heads, harpoons, and lances, and jealously watching his every sooty movement, as he toiled. Nevertheless, this old man's was a patient hammer wielded by a patient arm. No murmur, no impatience, no petulance did come from him. Silent, slow, and solemn; bowing over still further his chronically broken back, he toiled away, as if toil were life itself, and the heavy beating of his hammer the heavy beating of his heart. And so it was.—Most miserable! A peculiar walk in this old man, a certain slight but painful appearing yawing in his gait, had at an early period of the voyage excited the curiosity of the mariners. And to the importunity of their persisted questionings he had finally given in; and so it came to pass that every one now knew the shameful story of his wretched fate. Belated, and not innocently, one bitter winter's midnight, on the road running between two country towns, the blacksmith half-stupidly felt the deadly numbness stealing over him, and sought refuge in a leaning, dilapidated barn. The issue was, the loss of the extremities of both feet. Out of this revelation, part by part, at last came out the four acts of the gladness, and the one long, and as yet uncatastrophied fifth act of the grief of his life's drama. He was an old man, who, at the age of nearly sixty, had postponedly encountered that thing in sorrow's technicals called ruin. He had been an artisan of famed excellence, and with plenty to do; owned a house and garden; embraced a youthful, daughter-like, loving wife, and three blithe, ruddy children; every Sunday went to a cheerful-looking church, planted in a grove. But one night, under cover of darkness, and further concealed in a most cunning disguisement, a desperate burglar slid into his happy home, and robbed them all of everything. And darker yet to tell, the blacksmith himself did ignorantly conduct this burglar into his family's heart. It was the Bottle Conjuror! Upon the opening of that fatal cork, forth flew the fiend, and shrivelled up his home. Now, for prudent, most wise, and economic reasons, the blacksmith's shop was in the basement of his dwelling, but with a separate entrance to it; so that always had the young and loving healthy wife listened with no unhappy nervousness, but with vigorous pleasure, to the stout ringing of her young-armed old husband's hammer; whose reverberations, muffled by passing through the floors and walls, came up to her, not unsweetly, in her nursery; and so, to stout Labor's iron lullaby, the blacksmith's infants were rocked to slumber. Oh, woe on woe! Oh, Death, why canst thou not sometimes be timely? Hadst thou taken this old blacksmith to thyself ere his full ruin came upon him, then had the young widow had a delicious grief, and her orphans a truly venerable, legendary sire to dream of in their after years; and all of them a care-killing competency.
-
-
-
-
\ No newline at end of file
diff --git a/tests/test_cli.sh b/tests/test_cli.sh
deleted file mode 100755
index 03d6df8..0000000
--- a/tests/test_cli.sh
+++ /dev/null
@@ -1,175 +0,0 @@
-#!/bin/bash
-# Comprehensive CLI Testing Script
-# Tests all brightdata CLI commands to validate end-user experience
-
-set -e # Exit on error
-
-echo "================================================================================"
-echo "COMPREHENSIVE CLI VALIDATION - Testing Real User Experience"
-echo "================================================================================"
-echo "Timestamp: $(date '+%Y%m%d_%H%M%S')"
-echo "================================================================================"
-
-# Create probe directory structure for CLI tests
-PROBE_DIR="probe/cli"
-mkdir -p "$PROBE_DIR"/{scrape,search,help,errors}
-
-TIMESTAMP=$(date '+%Y%m%d_%H%M%S')
-SUMMARY_FILE="$PROBE_DIR/cli_summary_$TIMESTAMP.txt"
-
-# Track results
-TOTAL_TESTS=0
-PASSED_TESTS=0
-FAILED_TESTS=0
-
-# Helper function to run CLI test
-run_cli_test() {
- local test_name=$1
- local command=$2
- local category=$3
- local output_file="$PROBE_DIR/$category/${test_name}_${TIMESTAMP}.txt"
-
- TOTAL_TESTS=$((TOTAL_TESTS + 1))
-
- echo ""
- echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
- echo "TEST: $test_name"
- echo "COMMAND: $command"
- echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-
- # Run command and save output
- if eval "$command" > "$output_file" 2>&1; then
- echo " ✅ PASSED"
- echo " 📁 Output: $output_file"
- PASSED_TESTS=$((PASSED_TESTS + 1))
- return 0
- else
- EXIT_CODE=$?
- echo " ❌ FAILED (exit code: $EXIT_CODE)"
- echo " 📁 Error output: $output_file"
- FAILED_TESTS=$((FAILED_TESTS + 1))
- return 1
- fi
-}
-
-# =============================================================================
-# STEP 1: HELP COMMANDS
-# =============================================================================
-
-echo ""
-echo "📋 STEP 1: HELP & INFO COMMANDS"
-echo "================================================================================"
-
-run_cli_test "help_main" "brightdata --help" "help"
-run_cli_test "help_scrape" "brightdata scrape --help" "help"
-run_cli_test "help_search" "brightdata search --help" "help"
-run_cli_test "help_scrape_amazon" "brightdata scrape amazon --help" "help"
-run_cli_test "help_search_amazon" "brightdata search amazon --help" "help"
-run_cli_test "help_search_linkedin" "brightdata search linkedin --help" "help"
-
-# =============================================================================
-# STEP 2: SCRAPE COMMANDS (if we have test token - these will fail without real API)
-# =============================================================================
-
-echo ""
-echo "📋 STEP 2: SCRAPE COMMANDS (syntax validation)"
-echo "================================================================================"
-echo "Note: These test CLI syntax, not actual API calls (would need valid token)"
-
-# Test CLI syntax validation (will fail on auth but validates parsing)
-run_cli_test "scrape_amazon_products_help" \
- "brightdata scrape amazon products --help" \
- "scrape" || true
-
-run_cli_test "scrape_linkedin_profiles_help" \
- "brightdata scrape linkedin profiles --help" \
- "scrape" || true
-
-run_cli_test "scrape_facebook_posts_help" \
- "brightdata scrape facebook --help" \
- "scrape" || true
-
-run_cli_test "scrape_instagram_profiles_help" \
- "brightdata scrape instagram --help" \
- "scrape" || true
-
-# =============================================================================
-# STEP 3: SEARCH COMMANDS (syntax validation)
-# =============================================================================
-
-echo ""
-echo "📋 STEP 3: SEARCH COMMANDS (syntax validation)"
-echo "================================================================================"
-
-run_cli_test "search_google_help" \
- "brightdata search google --help" \
- "search" || true
-
-run_cli_test "search_linkedin_jobs_help" \
- "brightdata search linkedin jobs --help" \
- "search" || true
-
-# =============================================================================
-# STEP 4: FORMAT OPTIONS
-# =============================================================================
-
-echo ""
-echo "📋 STEP 4: OUTPUT FORMAT OPTIONS"
-echo "================================================================================"
-
-# Test that --output-format is recognized
-run_cli_test "format_json_help" \
- "brightdata scrape --help | grep 'output-format'" \
- "help" || true
-
-run_cli_test "format_generic_help" \
- "brightdata scrape generic --help" \
- "help" || true
-
-# =============================================================================
-# FINAL SUMMARY
-# =============================================================================
-
-echo ""
-echo "================================================================================"
-echo "CLI VALIDATION SUMMARY"
-echo "================================================================================"
-
-{
- echo "Timestamp: $(date)"
- echo ""
- echo "TEST RESULTS:"
- echo " Total: $TOTAL_TESTS"
- echo " Passed: $PASSED_TESTS"
- echo " Failed: $FAILED_TESTS"
- echo ""
-
- if [ $FAILED_TESTS -eq 0 ]; then
- echo "✅ ALL CLI TESTS PASSED"
- echo ""
- echo "CLI is fully functional and ready for users!"
- else
- echo "⚠️ $FAILED_TESTS test(s) failed"
- echo ""
- echo "Check probe/cli/ directory for details"
- fi
-
- echo ""
- echo "📁 All outputs saved to: probe/cli/"
- echo ""
- echo "Directory structure:"
- find "$PROBE_DIR" -type f | sort
-
-} | tee "$SUMMARY_FILE"
-
-echo ""
-echo "================================================================================"
-if [ $FAILED_TESTS -eq 0 ]; then
- echo "🎉 CLI VALIDATION COMPLETE - ALL SYSTEMS GO"
- exit 0
-else:
- echo "⚠️ SOME CLI TESTS FAILED - CHECK OUTPUTS"
- exit 1
-fi
-echo "================================================================================"
-
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
index e0310a0..e69de29 100644
--- a/tests/unit/__init__.py
+++ b/tests/unit/__init__.py
@@ -1 +0,0 @@
-"""Unit tests."""
diff --git a/tests/unit/test_amazon.py b/tests/unit/test_amazon.py
deleted file mode 100644
index 9a313d0..0000000
--- a/tests/unit/test_amazon.py
+++ /dev/null
@@ -1,314 +0,0 @@
-"""Unit tests for Amazon scraper."""
-
-from brightdata import BrightDataClient
-from brightdata.scrapers.amazon import AmazonScraper
-
-
-class TestAmazonScraperURLBased:
- """Test Amazon scraper (URL-based extraction)."""
-
- def test_amazon_scraper_has_products_method(self):
- """Test Amazon scraper has products method (async-first API)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "products")
- assert callable(scraper.products)
-
- def test_amazon_scraper_has_reviews_method(self):
- """Test Amazon scraper has reviews method (async-first API)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reviews")
- assert callable(scraper.reviews)
-
- def test_amazon_scraper_has_sellers_method(self):
- """Test Amazon scraper has sellers method (async-first API)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "sellers")
- assert callable(scraper.sellers)
-
- def test_products_method_signature(self):
- """Test products method has correct signature."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.products)
-
- # Required: url parameter
- assert "url" in sig.parameters
-
- # Optional: sync and timeout
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 240
-
- def test_reviews_method_signature(self):
- """Test reviews method has correct signature."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reviews)
-
- # Required: url
- assert "url" in sig.parameters
-
- # Optional filters
- assert "pastDays" in sig.parameters
- assert "keyWord" in sig.parameters
- assert "numOfReviews" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 240
-
- def test_sellers_method_signature(self):
- """Test sellers method has correct signature."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.sellers)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 240
-
-
-class TestAmazonDatasetIDs:
- """Test Amazon has correct dataset IDs."""
-
- def test_scraper_has_all_dataset_ids(self):
- """Test scraper has dataset IDs for all types."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert scraper.DATASET_ID # Products
- assert scraper.DATASET_ID_REVIEWS
- assert scraper.DATASET_ID_SELLERS
-
- # All should start with gd_
- assert scraper.DATASET_ID.startswith("gd_")
- assert scraper.DATASET_ID_REVIEWS.startswith("gd_")
- assert scraper.DATASET_ID_SELLERS.startswith("gd_")
-
- def test_dataset_ids_are_correct(self):
- """Test dataset IDs match Bright Data identifiers."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- # Verify known IDs
- assert scraper.DATASET_ID == "gd_l7q7dkf244hwjntr0" # Products
- assert scraper.DATASET_ID_REVIEWS == "gd_le8e811kzy4ggddlq" # Reviews
- assert scraper.DATASET_ID_SELLERS == "gd_lhotzucw1etoe5iw1k" # Sellers
-
-
-class TestAmazonSyncVsAsyncMode:
- """Test sync vs async mode handling."""
-
- def test_default_timeout_is_correct(self):
- """Test default timeout is 240s for async workflow."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.products)
-
- assert sig.parameters["timeout"].default == 240
-
- def test_all_methods_dont_have_sync_parameter(self):
- """Test all scrape methods don't have sync parameter (standard async pattern)."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- for method_name in ["products", "reviews", "sellers"]:
- sig = inspect.signature(getattr(scraper, method_name))
- assert "sync" not in sig.parameters
-
-
-class TestAmazonAPISpecCompliance:
- """Test compliance with exact API specifications."""
-
- def test_products_api_spec(self):
- """Test products() matches CP API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: client.scrape.amazon.products(url, timeout=240)
- import inspect
-
- sig = inspect.signature(client.scrape.amazon.products)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 240
-
- def test_reviews_api_spec(self):
- """Test reviews() matches CP API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: reviews(url, pastDays, keyWord, numOfReviews, sync, timeout)
- import inspect
-
- sig = inspect.signature(client.scrape.amazon.reviews)
-
- params = sig.parameters
- assert "url" in params
- assert "pastDays" in params
- assert "keyWord" in params
- assert "numOfReviews" in params
- assert "sync" not in params
- assert "timeout" in params
-
- def test_sellers_api_spec(self):
- """Test sellers() matches CP API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: sellers(url, timeout=240)
- import inspect
-
- sig = inspect.signature(client.scrape.amazon.sellers)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
-
-class TestAmazonParameterArraySupport:
- """Test array parameter support (str | array)."""
-
- def test_url_accepts_string(self):
- """Test url parameter accepts single string."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.products)
-
- # Type annotation should allow str | List[str]
- url_annotation = str(sig.parameters["url"].annotation)
- assert "Union" in url_annotation or "|" in url_annotation
- assert "str" in url_annotation
-
- def test_url_accepts_list(self):
- """Test url parameter accepts list."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.products)
-
- url_annotation = str(sig.parameters["url"].annotation)
- assert "List" in url_annotation or "list" in url_annotation
-
-
-class TestAmazonAsyncFirstAPI:
- """Test all methods follow async-first pattern."""
-
- def test_all_methods_exist(self):
- """Test all methods exist (async-first API, no _async suffix)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- methods = ["products", "reviews", "sellers"]
-
- for method in methods:
- assert hasattr(scraper, method)
- assert callable(getattr(scraper, method))
-
-
-class TestAmazonClientIntegration:
- """Test Amazon integrates properly with client."""
-
- def test_amazon_accessible_via_client(self):
- """Test Amazon scraper accessible via client.scrape.amazon."""
- client = BrightDataClient(token="test_token_123456789")
-
- amazon = client.scrape.amazon
- assert amazon is not None
- assert isinstance(amazon, AmazonScraper)
-
- def test_client_passes_token_to_scraper(self):
- """Test client passes token to Amazon scraper."""
- token = "test_token_123456789"
- client = BrightDataClient(token=token)
-
- amazon = client.scrape.amazon
- assert amazon.bearer_token == token
-
- def test_all_amazon_methods_accessible_through_client(self):
- """Test all Amazon methods accessible through client."""
- client = BrightDataClient(token="test_token_123456789")
-
- amazon = client.scrape.amazon
-
- assert callable(amazon.products)
- assert callable(amazon.reviews)
- assert callable(amazon.sellers)
-
-
-class TestAmazonReviewsFilters:
- """Test Amazon reviews method filters."""
-
- def test_reviews_accepts_pastDays_filter(self):
- """Test reviews method accepts pastDays parameter."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reviews)
-
- assert "pastDays" in sig.parameters
- assert sig.parameters["pastDays"].default is None # Optional
-
- def test_reviews_accepts_keyWord_filter(self):
- """Test reviews method accepts keyWord parameter."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reviews)
-
- assert "keyWord" in sig.parameters
- assert sig.parameters["keyWord"].default is None
-
- def test_reviews_accepts_numOfReviews_filter(self):
- """Test reviews method accepts numOfReviews parameter."""
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reviews)
-
- assert "numOfReviews" in sig.parameters
- assert sig.parameters["numOfReviews"].default is None
-
-
-class TestAmazonPhilosophicalPrinciples:
- """Test Amazon scraper follows philosophical principles."""
-
- def test_consistent_timeout_defaults(self):
- """Test consistent timeout defaults across methods."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- import inspect
-
- # All methods should default to 240s
- for method_name in ["products", "reviews", "sellers"]:
- sig = inspect.signature(getattr(scraper, method_name))
- assert sig.parameters["timeout"].default == 240
-
- def test_uses_standard_async_workflow(self):
- """Test methods use standard async workflow (no sync parameter)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- import inspect
-
- for method_name in ["products", "reviews", "sellers"]:
- sig = inspect.signature(getattr(scraper, method_name))
-
- # Should not have sync parameter
- assert "sync" not in sig.parameters
-
- def test_amazon_is_platform_expert(self):
- """Test Amazon scraper knows its platform."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "amazon"
- assert scraper.DATASET_ID # Has dataset knowledge
- assert scraper.MIN_POLL_TIMEOUT == 240 # Knows Amazon takes longer
diff --git a/tests/unit/test_async_unblocker.py b/tests/unit/test_async_unblocker.py
index 6595397..bc75ed1 100644
--- a/tests/unit/test_async_unblocker.py
+++ b/tests/unit/test_async_unblocker.py
@@ -1,190 +1,174 @@
-"""Unit tests for AsyncUnblockerClient."""
+"""Tests for web_unlocker/async_client.py — Trigger, status, and fetch operations."""
import pytest
from unittest.mock import AsyncMock, MagicMock
-from brightdata.api.async_unblocker import AsyncUnblockerClient
+
+from brightdata.web_unlocker.async_client import AsyncUnblockerClient
from brightdata.exceptions import APIError
+from tests.conftest import MockContextManager
+
-class MockAsyncContextManager:
- """Helper to mock async context managers."""
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
- def __init__(self, return_value):
- self.return_value = return_value
- async def __aenter__(self):
- return self.return_value
+@pytest.fixture
+def engine():
+ eng = MagicMock()
+ eng.BASE_URL = "https://api.brightdata.com"
+ return eng
- async def __aexit__(self, exc_type, exc_val, exc_tb):
- pass
+@pytest.fixture
+def client(engine):
+ return AsyncUnblockerClient(engine)
-class TestAsyncUnblockerClient:
- """Test AsyncUnblockerClient functionality."""
- def setup_method(self):
- """Set up test fixtures."""
- self.engine = MagicMock()
- self.engine.BASE_URL = "https://api.brightdata.com"
- self.client = AsyncUnblockerClient(self.engine)
+# ---------------------------------------------------------------------------
+# Trigger
+# ---------------------------------------------------------------------------
+
+class TestTrigger:
@pytest.mark.asyncio
- async def test_trigger_success(self):
- """Test successful trigger returns response_id from header."""
- # Mock response with x-response-id header
+ async def test_returns_response_id(self, client, engine):
response = MagicMock()
response.headers.get.return_value = "test_response_id_123"
+ engine.post_to_url = MagicMock(return_value=MockContextManager(response))
- # Mock post_to_url to return async context manager
- self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response))
-
- # Trigger request
- response_id = await self.client.trigger(zone="test_zone", url="https://example.com")
+ response_id = await client.trigger(zone="test_zone", url="https://example.com")
- # Verify response_id returned
assert response_id == "test_response_id_123"
-
- # Verify correct endpoint called
- self.engine.post_to_url.assert_called_once()
- call_args = self.engine.post_to_url.call_args
+ engine.post_to_url.assert_called_once()
+ call_args = engine.post_to_url.call_args
assert call_args[0][0] == "https://api.brightdata.com/unblocker/req"
assert call_args[1]["params"] == {"zone": "test_zone"}
assert call_args[1]["json_data"]["url"] == "https://example.com"
@pytest.mark.asyncio
- async def test_trigger_with_additional_params(self):
- """Test trigger passes additional parameters correctly."""
+ async def test_passes_additional_params(self, client, engine):
response = MagicMock()
response.headers.get.return_value = "response_id_456"
+ engine.post_to_url = MagicMock(return_value=MockContextManager(response))
- self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response))
-
- # Trigger with additional params
- response_id = await self.client.trigger(
+ response_id = await client.trigger(
zone="my_zone", url="https://google.com/search?q=test", format="raw", country="US"
)
assert response_id == "response_id_456"
-
- # Verify params merged into payload
- call_args = self.engine.post_to_url.call_args
- payload = call_args[1]["json_data"]
+ payload = engine.post_to_url.call_args[1]["json_data"]
assert payload["url"] == "https://google.com/search?q=test"
assert payload["format"] == "raw"
assert payload["country"] == "US"
@pytest.mark.asyncio
- async def test_trigger_no_response_id(self):
- """Test trigger returns None when no response_id header."""
+ async def test_returns_none_when_no_response_id(self, client, engine):
response = MagicMock()
- response.headers.get.return_value = None # No x-response-id
+ response.headers.get.return_value = None
+ engine.post_to_url = MagicMock(return_value=MockContextManager(response))
- self.engine.post_to_url = MagicMock(return_value=MockAsyncContextManager(response))
+ response_id = await client.trigger(zone="test_zone", url="https://example.com")
+ assert response_id is None
- response_id = await self.client.trigger(zone="test_zone", url="https://example.com")
- assert response_id is None
+# ---------------------------------------------------------------------------
+# Get Status
+# ---------------------------------------------------------------------------
+
+class TestGetStatus:
@pytest.mark.asyncio
- async def test_get_status_ready(self):
- """Test get_status returns 'ready' for HTTP 200."""
+ async def test_200_returns_ready(self, client, engine):
response = MagicMock()
response.status = 200
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
-
- status = await self.client.get_status(zone="test_zone", response_id="abc123")
+ status = await client.get_status(zone="test_zone", response_id="abc123")
assert status == "ready"
-
- # Verify correct endpoint and params
- call_args = self.engine.get_from_url.call_args
+ call_args = engine.get_from_url.call_args
assert call_args[0][0] == "https://api.brightdata.com/unblocker/get_result"
assert call_args[1]["params"]["zone"] == "test_zone"
assert call_args[1]["params"]["response_id"] == "abc123"
@pytest.mark.asyncio
- async def test_get_status_pending(self):
- """Test get_status returns 'pending' for HTTP 202."""
+ async def test_202_returns_pending(self, client, engine):
response = MagicMock()
response.status = 202
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
-
- status = await self.client.get_status(zone="test_zone", response_id="xyz789")
-
+ status = await client.get_status(zone="test_zone", response_id="xyz789")
assert status == "pending"
@pytest.mark.asyncio
- async def test_get_status_error(self):
- """Test get_status returns 'error' for non-200/202 status."""
- # Test various error codes
+ async def test_error_codes_return_error(self, client, engine):
for error_code in [400, 404, 500, 503]:
response = MagicMock()
response.status = error_code
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
+ status = await client.get_status(zone="test_zone", response_id="err123")
+ assert status == "error", f"Expected 'error' for HTTP {error_code}"
- status = await self.client.get_status(zone="test_zone", response_id="err123")
- assert status == "error", f"Expected 'error' for HTTP {error_code}"
+# ---------------------------------------------------------------------------
+# Fetch Result
+# ---------------------------------------------------------------------------
+
+class TestFetchResult:
@pytest.mark.asyncio
- async def test_fetch_result_success(self):
- """Test fetch_result returns parsed JSON for HTTP 200."""
+ async def test_200_returns_json(self, client, engine):
expected_data = {"general": {"search_engine": "google"}, "organic": [{"title": "Result 1"}]}
-
response = MagicMock()
response.status = 200
response.json = AsyncMock(return_value=expected_data)
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
-
- data = await self.client.fetch_result(zone="test_zone", response_id="fetch123")
+ data = await client.fetch_result(zone="test_zone", response_id="fetch123")
assert data == expected_data
response.json.assert_called_once()
@pytest.mark.asyncio
- async def test_fetch_result_not_ready(self):
- """Test fetch_result raises APIError for HTTP 202 (pending)."""
+ async def test_202_raises_api_error(self, client, engine):
response = MagicMock()
response.status = 202
-
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
with pytest.raises(APIError) as exc_info:
- await self.client.fetch_result(zone="test_zone", response_id="pending123")
+ await client.fetch_result(zone="test_zone", response_id="pending123")
assert "not ready yet" in str(exc_info.value).lower()
assert "202" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_fetch_result_error(self):
- """Test fetch_result raises APIError for error status codes."""
+ async def test_500_raises_api_error(self, client, engine):
response = MagicMock()
response.status = 500
response.text = AsyncMock(return_value="Internal Server Error")
-
- self.engine.get_from_url = MagicMock(return_value=MockAsyncContextManager(response))
+ engine.get_from_url = MagicMock(return_value=MockContextManager(response))
with pytest.raises(APIError) as exc_info:
- await self.client.fetch_result(zone="test_zone", response_id="error123")
+ await client.fetch_result(zone="test_zone", response_id="error123")
- error_msg = str(exc_info.value)
- assert "500" in error_msg
- assert "Internal Server Error" in error_msg
+ assert "500" in str(exc_info.value)
+ assert "Internal Server Error" in str(exc_info.value)
- @pytest.mark.asyncio
- async def test_endpoint_constants(self):
- """Test that endpoint constants are correct."""
- assert self.client.TRIGGER_ENDPOINT == "/unblocker/req"
- assert self.client.FETCH_ENDPOINT == "/unblocker/get_result"
- @pytest.mark.asyncio
- async def test_client_initialization(self):
- """Test client initializes with AsyncEngine."""
- engine = MagicMock()
- client = AsyncUnblockerClient(engine)
+# ---------------------------------------------------------------------------
+# Constants and init
+# ---------------------------------------------------------------------------
- assert client.engine is engine
+
+class TestClientSetup:
+ def test_endpoint_constants(self, client):
+ assert client.TRIGGER_ENDPOINT == "/unblocker/req"
+ assert client.FETCH_ENDPOINT == "/unblocker/get_result"
+
+ def test_stores_engine_reference(self):
+ engine = MagicMock()
+ c = AsyncUnblockerClient(engine)
+ assert c.engine is engine
diff --git a/tests/unit/test_batch.py b/tests/unit/test_batch.py
deleted file mode 100644
index eef2da9..0000000
--- a/tests/unit/test_batch.py
+++ /dev/null
@@ -1,172 +0,0 @@
-"""
-Tests for batch scraping operations.
-
-Verifies that scraping multiple URLs returns List[ScrapeResult] correctly.
-"""
-
-from brightdata import BrightDataClient
-
-
-class TestBatchOperations:
- """Test batch scraping returns correct types."""
-
- def test_single_url_returns_single_result(self):
- """Test that a single URL returns ScrapeResult (not list)."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Verify single URL behavior
- scraper = client.scrape.amazon
-
- # Single URL should return ScrapeResult
- import inspect
-
- sig = inspect.signature(scraper.products)
-
- # Should accept Union[str, List[str]]
- params = sig.parameters
- assert "url" in params
-
- def test_list_with_one_url_returns_single_result(self):
- """Test that list with 1 URL returns unwrapped ScrapeResult."""
- # This is the expected behavior - list with 1 item gets unwrapped
- # This test documents the API contract
- pass
-
- def test_multiple_urls_should_return_list(self):
- """Test that multiple URLs should return List[ScrapeResult]."""
- # This documents that the API SHOULD return a list of results
- # when given multiple URLs, not a single result with data as list
-
- # Expected behavior:
- # Input: ["url1", "url2", "url3"]
- # Output: [ScrapeResult, ScrapeResult, ScrapeResult]
- # NOT: ScrapeResult with data=[item1, item2, item3]
- pass
-
- def test_batch_result_type_annotations(self):
- """Test that method signatures indicate Union[ScrapeResult, List[ScrapeResult]]."""
- from brightdata.scrapers.amazon import AmazonScraper
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- import inspect
-
- sig = inspect.signature(scraper.products)
-
- # Check return type annotation
- return_type = sig.return_annotation
- assert return_type != inspect.Signature.empty, "Should have return type annotation"
-
- # Should be Union[ScrapeResult, List[ScrapeResult]]
- type_str = str(return_type)
- assert "ScrapeResult" in type_str
- assert "List" in type_str or "Union" in type_str
-
-
-class TestBatchScrapingBehavior:
- """Test actual batch scraping behavior."""
-
- def test_batch_operations_contract(self):
- """Document the batch operations API contract."""
- # API Contract:
- # 1. Single URL string → ScrapeResult
- # 2. List with 1 URL → ScrapeResult (unwrapped for convenience)
- # 3. List with 2+ URLs → List[ScrapeResult] (one per URL)
-
- # This ensures each URL gets its own result object with:
- # - Individual success/error status
- # - Individual timing information
- # - Individual cost tracking
- # - Individual data payload
- pass
-
- def test_batch_result_independence(self):
- """Test that batch results are independent."""
- # Each result in a batch should be independent:
- # - If URL 1 fails, URL 2 should still have its own result
- # - Each result has its own cost calculation
- # - Each result has its own timing data
- # - Each result has its own url field set
- pass
-
-
-class TestBatchErrorHandling:
- """Test batch operations error handling."""
-
- def test_batch_with_mixed_success_failure(self):
- """Test batch operations with some URLs succeeding and some failing."""
- # Expected: Each URL gets its own ScrapeResult
- # Some have success=True, some have success=False
- # All are in the returned list
- pass
-
- def test_batch_cost_calculation(self):
- """Test that costs are divided among batch results."""
- # If total cost is $0.003 for 3 URLs
- # Each result should have cost=$0.001
- pass
-
-
-class TestBatchImplementationAllPlatforms:
- """Verify batch fix is implemented across ALL platforms."""
-
- def test_amazon_has_batch_logic(self):
- """Verify Amazon scraper has batch transformation logic."""
- import inspect
- from brightdata.scrapers.amazon import AmazonScraper
-
- source = inspect.getsource(AmazonScraper)
-
- # Should have the batch transformation code
- assert "elif not is_single and isinstance(result.data, list):" in source
- assert "for url_item, data_item in zip" in source
- assert "List[ScrapeResult]" in source or "results.append" in source
-
- def test_linkedin_has_batch_logic(self):
- """Verify LinkedIn scraper has batch transformation logic."""
- import inspect
- from brightdata.scrapers.linkedin import LinkedInScraper
-
- source = inspect.getsource(LinkedInScraper)
-
- assert "elif not is_single and isinstance(result.data, list):" in source
- assert "for url_item, data_item in zip" in source
-
- def test_instagram_has_batch_logic(self):
- """Verify Instagram scraper has batch transformation logic."""
- import inspect
- from brightdata.scrapers.instagram import InstagramScraper
-
- source = inspect.getsource(InstagramScraper)
-
- assert "elif not is_single and isinstance(result.data, list):" in source
- assert "for url_item, data_item in zip" in source
-
- def test_facebook_has_batch_logic(self):
- """Verify Facebook scraper has batch transformation logic."""
- import inspect
- from brightdata.scrapers.facebook import FacebookScraper
-
- source = inspect.getsource(FacebookScraper)
-
- assert "elif not is_single and isinstance(result.data, list):" in source
- assert "for url_item, data_item in zip" in source
-
-
-class TestBatchBugRegression:
- """Ensure the batch bug doesn't regress."""
-
- def test_batch_returns_list_not_single_result_with_list_data(self):
- """THE KEY TEST: Batch operations must return List[ScrapeResult], not ScrapeResult with list data."""
- # This is the core issue from issues.md
- #
- # BEFORE (BUG):
- # Input: ["url1", "url2"]
- # Output: ScrapeResult(data=[item1, item2]) ❌ WRONG
- #
- # AFTER (FIXED):
- # Input: ["url1", "url2"]
- # Output: [ScrapeResult(data=item1), ScrapeResult(data=item2)] ✅ CORRECT
- #
- # The fix ensures each URL gets its own ScrapeResult object
- assert True # Implementation verified by code inspection tests above
diff --git a/tests/unit/test_chatgpt.py b/tests/unit/test_chatgpt.py
deleted file mode 100644
index 9fd9e2a..0000000
--- a/tests/unit/test_chatgpt.py
+++ /dev/null
@@ -1,265 +0,0 @@
-"""Unit tests for ChatGPT search service."""
-
-import inspect
-from brightdata import BrightDataClient
-from brightdata.scrapers.chatgpt import ChatGPTSearchService
-
-
-class TestChatGPTSearchService:
- """Test ChatGPT search service."""
-
- def test_chatgpt_search_has_chatGPT_method(self):
- """Test ChatGPT search has chatGPT method (async-first API)."""
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- assert hasattr(search, "chatGPT")
- assert callable(search.chatGPT)
-
- def test_chatGPT_method_signature(self):
- """Test chatGPT method has correct signature."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- # Required: prompt
- assert "prompt" in sig.parameters
-
- # Optional parameters
- assert "country" in sig.parameters
- assert "secondaryPrompt" in sig.parameters
- assert "webSearch" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 180
-
- def test_chatGPT_validates_required_prompt(self):
- """Test chatGPT raises error if prompt is missing."""
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- # This would fail at runtime, but we test the validation exists
- # (Can't actually call without mocking the engine)
- assert "prompt" in str(inspect.signature(search.chatGPT).parameters)
-
-
-class TestChatGPTAPISpecCompliance:
- """Test compliance with exact API specifications."""
-
- def test_api_spec_matches_cp_link(self):
- """Test method matches CP link specification."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: client.search.chatGPT(prompt, country, secondaryPrompt, webSearch, timeout)
- import inspect
-
- sig = inspect.signature(client.search.chatGPT.chatGPT)
-
- params = sig.parameters
-
- # All parameters from spec
- assert "prompt" in params # str | array, required
- assert "country" in params # str | array, 2-letter format
- assert "secondaryPrompt" in params # str | array
- assert "webSearch" in params # bool | array
- assert "sync" not in params # Removed - uses standard async workflow
- assert "timeout" in params # int, default: 180
-
- def test_parameter_defaults_match_spec(self):
- """Test parameter defaults match specification."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- # Defaults per spec
- assert sig.parameters["timeout"].default == 180
-
- # Optional params should default to None
- assert sig.parameters["country"].default is None
- assert sig.parameters["secondaryPrompt"].default is None
- assert sig.parameters["webSearch"].default is None
-
-
-class TestChatGPTParameterArraySupport:
- """Test array parameter support (str | array, bool | array)."""
-
- def test_prompt_accepts_string(self):
- """Test prompt parameter accepts single string."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- # Type annotation should allow str | List[str]
- prompt_annotation = str(sig.parameters["prompt"].annotation)
- assert "Union" in prompt_annotation or "str" in prompt_annotation
-
- def test_prompt_accepts_list(self):
- """Test prompt parameter accepts list."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- prompt_annotation = str(sig.parameters["prompt"].annotation)
- assert "List" in prompt_annotation or "list" in prompt_annotation
-
- def test_country_accepts_string_or_list(self):
- """Test country accepts str | list."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- annotation = str(sig.parameters["country"].annotation)
- # Should be Optional[Union[str, List[str]]]
- assert "str" in annotation
-
- def test_webSearch_accepts_bool_or_list(self):
- """Test webSearch accepts bool | list[bool]."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- annotation = str(sig.parameters["webSearch"].annotation)
- # Should accept bool | List[bool]
- assert "bool" in annotation
-
-
-class TestChatGPTSyncAsyncMode:
- """Test standard async workflow (no sync parameter)."""
-
- def test_no_sync_parameter(self):
- """Test methods don't have sync parameter (standard async pattern)."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- assert "sync" not in sig.parameters
-
- def test_timeout_defaults_to_180(self):
- """Test timeout defaults to 180."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
- sig = inspect.signature(search.chatGPT)
-
- assert sig.parameters["timeout"].default == 180
-
- def test_has_chatGPT_method(self):
- """Test has chatGPT method (async-first API)."""
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- assert hasattr(search, "chatGPT")
- assert callable(search.chatGPT)
-
-
-class TestChatGPTClientIntegration:
- """Test ChatGPT search integrates with client."""
-
- def test_chatgpt_accessible_via_client_search(self):
- """Test ChatGPT search accessible via client.search.chatGPT."""
- client = BrightDataClient(token="test_token_123456789")
-
- chatgpt = client.search.chatGPT
- assert chatgpt is not None
- assert isinstance(chatgpt, ChatGPTSearchService)
-
- def test_client_passes_token_to_chatgpt_search(self):
- """Test client passes token to ChatGPT search."""
- token = "test_token_123456789"
- client = BrightDataClient(token=token)
-
- chatgpt = client.search.chatGPT
- assert chatgpt.bearer_token == token
-
- def test_chatGPT_method_callable_through_client(self):
- """Test chatGPT method callable through client (async-first API)."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Should be able to access the method
- assert callable(client.search.chatGPT.chatGPT)
-
-
-class TestChatGPTInterfaceExamples:
- """Test interface examples from specification."""
-
- def test_single_prompt_interface(self):
- """Test single prompt interface."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Interface should accept single prompt
- import inspect
-
- sig = inspect.signature(client.search.chatGPT.chatGPT)
-
- # Can call with just prompt
- assert "prompt" in sig.parameters
-
- # Other params are optional
- assert sig.parameters["country"].default is None
- assert sig.parameters["secondaryPrompt"].default is None
- assert sig.parameters["webSearch"].default is None
-
- def test_batch_prompts_interface(self):
- """Test batch prompts interface."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Should accept lists for all parameters
- import inspect
-
- sig = inspect.signature(client.search.chatGPT.chatGPT)
-
- # All array parameters should be in Union with List
- prompt_annotation = str(sig.parameters["prompt"].annotation)
- assert "List" in prompt_annotation
-
-
-class TestChatGPTCountryValidation:
- """Test country code validation."""
-
- def test_country_should_be_2_letter_format(self):
- """Test country parameter expects 2-letter format."""
- # This is validated in the implementation
- # We verify the docstring mentions it
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- # Check docstring mentions 2-letter format (async-first API)
- doc = search.chatGPT.__doc__
- assert doc is not None and (
- "2-letter" in doc or "2 letter" in doc.replace("-", " ") or "country" in doc.lower()
- )
-
-
-class TestChatGPTPhilosophicalPrinciples:
- """Test ChatGPT search follows philosophical principles."""
-
- def test_fixed_url_per_spec(self):
- """Test URL is fixed to chatgpt.com per spec."""
- # Per spec comment: "the param URL will be fixed to https://chatgpt.com"
- # This is handled in the implementation
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- # Verify implementation exists (can't test without API call)
- assert search.DATASET_ID == "gd_m7aof0k82r803d5bjm"
-
- def test_consistent_with_other_search_services(self):
- """Test ChatGPT search follows same patterns as other search services."""
- import inspect
-
- search = ChatGPTSearchService(bearer_token="test_token_123456789")
-
- # Should have chatGPT method (async-first API)
- assert hasattr(search, "chatGPT")
- assert callable(search.chatGPT)
-
- # Should have timeout parameter
- sig = inspect.signature(search.chatGPT)
- assert "timeout" in sig.parameters
-
- # Should not have sync parameter (standard async pattern)
- assert "sync" not in sig.parameters
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index 6445f6a..0d72ea9 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -1,233 +1,423 @@
-"""Unit tests for BrightDataClient."""
+"""Tests for client.py — BrightDataClient init, services, context manager, API methods."""
-import os
import pytest
-from unittest.mock import patch
-from brightdata import BrightDataClient
-from brightdata.exceptions import ValidationError
+from unittest.mock import AsyncMock, MagicMock, patch
+from brightdata.client import BrightDataClient
+from brightdata.exceptions import ValidationError, AuthenticationError, APIError
+from brightdata.scrapers.service import ScrapeService
+from brightdata.serp.service import SearchService
+from brightdata.crawler.service import CrawlerService
+from brightdata.datasets import DatasetsClient
-class TestClientInitialization:
- """Test client initialization and configuration."""
+from tests.conftest import MockResponse, MockContextManager
- def test_client_with_explicit_token(self):
- """Test client initialization with explicit token."""
- client = BrightDataClient(token="test_token_123456789")
- assert client.token == "test_token_123456789"
- assert client.timeout == 30 # Default timeout
- assert client.web_unlocker_zone == "sdk_unlocker"
- assert client.serp_zone == "sdk_serp"
- assert client.browser_zone == "sdk_browser"
+# ---------------------------------------------------------------------------
+# Token loading
+# ---------------------------------------------------------------------------
- def test_client_with_custom_config(self):
- """Test client with custom configuration."""
- client = BrightDataClient(
- token="custom_token_123456789",
- timeout=60,
- web_unlocker_zone="my_unlocker",
- serp_zone="my_serp",
- browser_zone="my_browser",
- )
- assert client.timeout == 60
- assert client.web_unlocker_zone == "my_unlocker"
- assert client.serp_zone == "my_serp"
- assert client.browser_zone == "my_browser"
-
- def test_client_loads_from_brightdata_api_token(self):
- """Test client loads token from BRIGHTDATA_API_TOKEN."""
- with patch.dict(os.environ, {"BRIGHTDATA_API_TOKEN": "env_token_123456789"}):
- client = BrightDataClient()
- assert client.token == "env_token_123456789"
-
- def test_client_prioritizes_explicit_token_over_env(self):
- """Test explicit token takes precedence over environment."""
- with patch.dict(os.environ, {"BRIGHTDATA_API_TOKEN": "env_token_123456789"}):
- client = BrightDataClient(token="explicit_token_123456789")
- assert client.token == "explicit_token_123456789"
-
- def test_client_raises_error_without_token(self):
- """Test client raises ValidationError when no token provided."""
- with patch.dict(os.environ, {}, clear=True):
- with pytest.raises(ValidationError) as exc_info:
- BrightDataClient()
+class TestTokenLoading:
+ def test_accepts_explicit_token(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.token == "tok_1234567890"
+
+ def test_strips_whitespace(self):
+ c = BrightDataClient(token=" tok_1234567890 ")
+ assert c.token == "tok_1234567890"
- assert "API token required" in str(exc_info.value)
- assert "BRIGHTDATA_API_TOKEN" in str(exc_info.value)
+ @patch.dict("os.environ", {"BRIGHTDATA_API_TOKEN": "env_token_12345"})
+ def test_reads_from_env(self):
+ c = BrightDataClient()
+ assert c.token == "env_token_12345"
- def test_client_raises_error_for_invalid_token_format(self):
- """Test client raises ValidationError for invalid token format."""
- with pytest.raises(ValidationError) as exc_info:
+ def test_raises_without_token(self):
+ with patch.dict("os.environ", {}, clear=True):
+ with pytest.raises(ValidationError, match="API token required"):
+ BrightDataClient()
+
+ def test_rejects_short_token(self):
+ with pytest.raises(ValidationError, match="at least 10 characters"):
BrightDataClient(token="short")
- assert "Invalid token format" in str(exc_info.value)
+ def test_rejects_non_string_token(self):
+ with pytest.raises(ValidationError):
+ BrightDataClient(token=12345678901) # type: ignore
- def test_client_raises_error_for_non_string_token(self):
- """Test client raises ValidationError for non-string token."""
- with pytest.raises(ValidationError) as exc_info:
- BrightDataClient(token=12345)
+ def test_explicit_token_takes_precedence(self):
+ with patch.dict("os.environ", {"BRIGHTDATA_API_TOKEN": "env_token_12345"}):
+ c = BrightDataClient(token="explicit_token_12345")
+ assert c.token == "explicit_token_12345"
- assert "Invalid token format" in str(exc_info.value)
+# ---------------------------------------------------------------------------
+# Init configuration
+# ---------------------------------------------------------------------------
-class TestClientTokenManagement:
- """Test token management and validation."""
- def test_token_is_stripped(self):
- """Test token whitespace is stripped."""
- client = BrightDataClient(token=" token_with_spaces_123 ")
- assert client.token == "token_with_spaces_123"
+class TestInitConfig:
+ def test_default_timeout(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.timeout == 30
- def test_env_token_is_stripped(self):
- """Test environment token whitespace is stripped."""
- with patch.dict(os.environ, {"BRIGHTDATA_API_TOKEN": " env_token_123456789 "}):
- client = BrightDataClient()
- assert client.token == "env_token_123456789"
+ def test_custom_timeout(self):
+ c = BrightDataClient(token="tok_1234567890", timeout=120)
+ assert c.timeout == 120
+ def test_default_zone_names(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.web_unlocker_zone == "sdk_unlocker"
+ assert c.serp_zone == "sdk_serp"
-class TestClientServiceProperties:
- """Test hierarchical service access properties."""
+ def test_custom_zone_names(self):
+ c = BrightDataClient(
+ token="tok_1234567890",
+ web_unlocker_zone="my_unlocker",
+ serp_zone="my_serp",
+ )
+ assert c.web_unlocker_zone == "my_unlocker"
+ assert c.serp_zone == "my_serp"
- def test_scrape_service_property(self):
- """Test scrape service property returns ScrapeService."""
- client = BrightDataClient(token="test_token_123456789")
+ def test_creates_engine(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.engine is not None
+ assert c.engine.bearer_token == "tok_1234567890"
- scrape_service = client.scrape
- assert scrape_service is not None
+ def test_services_none_before_access(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c._scrape_service is None
+ assert c._search_service is None
+ assert c._crawler_service is None
+ assert c._datasets_client is None
- # All scrapers should now work
- assert scrape_service.amazon is not None
- assert scrape_service.linkedin is not None
- assert scrape_service.chatgpt is not None
+ def test_auto_create_zones_default_true(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.auto_create_zones is True
- def test_scrape_service_is_cached(self):
- """Test scrape service is cached (returns same instance)."""
- client = BrightDataClient(token="test_token_123456789")
+ def test_auto_create_zones_can_disable(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ assert c.auto_create_zones is False
- service1 = client.scrape
- service2 = client.scrape
- assert service1 is service2
- def test_search_service_property(self):
- """Test search service property returns SearchService."""
- client = BrightDataClient(token="test_token_123456789")
+# ---------------------------------------------------------------------------
+# Service properties (lazy init)
+# ---------------------------------------------------------------------------
- search_service = client.search
- assert search_service is not None
- # All search methods should exist and be callable (async-first API)
- assert callable(search_service.google)
- assert callable(search_service.bing)
- assert callable(search_service.yandex)
+class TestServiceProperties:
+ def test_scrape_returns_scrape_service(self):
+ c = BrightDataClient(token="tok_1234567890")
+ s = c.scrape
+ assert isinstance(s, ScrapeService)
- def test_crawler_service_property(self):
- """Test crawler service property returns CrawlerService."""
- client = BrightDataClient(token="test_token_123456789")
+ def test_scrape_returns_same_instance(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.scrape is c.scrape
- crawler_service = client.crawler
- assert crawler_service is not None
- assert hasattr(crawler_service, "discover")
- assert hasattr(crawler_service, "sitemap")
+ def test_search_returns_search_service(self):
+ c = BrightDataClient(token="tok_1234567890")
+ s = c.search
+ assert isinstance(s, SearchService)
+ def test_search_returns_same_instance(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.search is c.search
-class TestClientBackwardCompatibility:
- """Test backward compatibility with old API."""
+ def test_crawler_returns_crawler_service(self):
+ c = BrightDataClient(token="tok_1234567890")
+ s = c.crawler
+ assert isinstance(s, CrawlerService)
- def test_scrape_url_method_exists(self):
- """Test scrape_url method exists for backward compatibility."""
- client = BrightDataClient(token="test_token_123456789")
- assert hasattr(client, "scrape_url")
+ def test_crawler_returns_same_instance(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.crawler is c.crawler
+ def test_datasets_returns_datasets_client(self):
+ c = BrightDataClient(token="tok_1234567890")
+ d = c.datasets
+ assert isinstance(d, DatasetsClient)
-class TestClientRepr:
- """Test client string representation."""
+ def test_datasets_returns_same_instance(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert c.datasets is c.datasets
- def test_repr_shows_token_preview(self):
- """Test __repr__ shows token preview."""
- client = BrightDataClient(token="1234567890abcdefghij")
- repr_str = repr(client)
- assert "BrightDataClient" in repr_str
- assert "1234567890" in repr_str # First 10 chars
- assert "fghij" in repr_str # Last 5 chars
- assert "abcde" not in repr_str # Middle should not be shown
+# ---------------------------------------------------------------------------
+# Browser property
+# ---------------------------------------------------------------------------
- def test_repr_shows_status(self):
- """Test __repr__ shows connection status."""
- client = BrightDataClient(token="test_token_123456789")
- repr_str = repr(client)
- assert "status" in repr_str.lower()
+class TestBrowserProperty:
+ def test_raises_without_credentials(self):
+ with patch.dict("os.environ", {}, clear=False):
+ # Ensure env vars are not set
+ import os
+ os.environ.pop("BRIGHTDATA_BROWSERAPI_USERNAME", None)
+ os.environ.pop("BRIGHTDATA_BROWSERAPI_PASSWORD", None)
-class TestClientConfiguration:
- """Test client configuration options."""
+ c = BrightDataClient(token="tok_1234567890")
+ with pytest.raises(ValidationError, match="Browser API credentials"):
+ _ = c.browser
+
+ def test_accepts_explicit_credentials(self):
+ c = BrightDataClient(
+ token="tok_1234567890",
+ browser_username="brd-user",
+ browser_password="pass123",
+ )
+ b = c.browser
+ assert b is not None
+
+ @patch.dict(
+ "os.environ",
+ {
+ "BRIGHTDATA_BROWSERAPI_USERNAME": "env-user",
+ "BRIGHTDATA_BROWSERAPI_PASSWORD": "env-pass",
+ },
+ )
+ def test_reads_credentials_from_env(self):
+ c = BrightDataClient(token="tok_1234567890")
+ b = c.browser
+ assert b is not None
+
+ def test_returns_same_instance(self):
+ c = BrightDataClient(
+ token="tok_1234567890",
+ browser_username="brd-user",
+ browser_password="pass123",
+ )
+ assert c.browser is c.browser
- def test_auto_create_zones_default_true(self):
- """Test auto_create_zones defaults to True."""
- client = BrightDataClient(token="test_token_123456789")
- assert client.auto_create_zones is True
-
- def test_auto_create_zones_can_be_enabled(self):
- """Test auto_create_zones can be enabled."""
- client = BrightDataClient(token="test_token_123456789", auto_create_zones=True)
- assert client.auto_create_zones is True
-
- def test_zones_ensured_flag_starts_false(self):
- """Test _zones_ensured flag starts as False."""
- client = BrightDataClient(token="test_token_123456789")
- assert client._zones_ensured is False
-
- def test_zone_manager_starts_as_none(self):
- """Test zone manager starts as None."""
- client = BrightDataClient(token="test_token_123456789")
- assert client._zone_manager is None
-
- def test_default_timeout_is_30(self):
- """Test default timeout is 30 seconds."""
- client = BrightDataClient(token="test_token_123456789")
- assert client.timeout == 30
-
- def test_custom_timeout_is_respected(self):
- """Test custom timeout is respected."""
- client = BrightDataClient(token="test_token_123456789", timeout=120)
- assert client.timeout == 120
-
-
-class TestClientErrorMessages:
- """Test client error messages are clear and helpful."""
-
- def test_missing_token_error_is_helpful(self):
- """Test missing token error provides helpful guidance."""
- with patch.dict(os.environ, {}, clear=True):
- with pytest.raises(ValidationError) as exc_info:
- BrightDataClient()
- error_msg = str(exc_info.value)
- assert "API token required" in error_msg
- assert "BrightDataClient(token=" in error_msg
- assert "BRIGHTDATA_API_TOKEN" in error_msg
- assert "https://brightdata.com" in error_msg
+# ---------------------------------------------------------------------------
+# _ensure_initialized
+# ---------------------------------------------------------------------------
- def test_invalid_token_format_error_is_clear(self):
- """Test invalid token format error is clear."""
- with pytest.raises(ValidationError) as exc_info:
- BrightDataClient(token="bad")
- error_msg = str(exc_info.value)
- assert "Invalid token format" in error_msg
- assert "at least 10 characters" in error_msg
+class TestEnsureInitialized:
+ def test_raises_if_no_session(self):
+ c = BrightDataClient(token="tok_1234567890")
+ with pytest.raises(RuntimeError, match="not initialized"):
+ c._ensure_initialized()
-class TestClientContextManager:
- """Test client context manager support."""
+# ---------------------------------------------------------------------------
+# Context manager
+# ---------------------------------------------------------------------------
- def test_client_supports_async_context_manager(self):
- """Test client supports async context manager protocol."""
- client = BrightDataClient(token="test_token_123456789")
- assert hasattr(client, "__aenter__")
- assert hasattr(client, "__aexit__")
- assert callable(client.__aenter__)
- assert callable(client.__aexit__)
+class TestContextManager:
+ @pytest.mark.asyncio
+ async def test_aenter_creates_session(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ assert c.engine._session is not None
+
+ @pytest.mark.asyncio
+ async def test_aexit_closes_session(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ session = c.engine._session
+ assert c.engine._session is None
+ assert session.closed
+
+ @pytest.mark.asyncio
+ async def test_returns_self(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c as client:
+ assert client is c
+
+ @pytest.mark.asyncio
+ async def test_validate_token_on_enter_success(self):
+ c = BrightDataClient(
+ token="tok_1234567890",
+ validate_token=True,
+ auto_create_zones=False,
+ )
+ with patch.object(c, "test_connection", new_callable=AsyncMock, return_value=True):
+ with patch.object(c, "_ensure_zones", new_callable=AsyncMock):
+ async with c:
+ pass # should not raise
+
+ @pytest.mark.asyncio
+ async def test_validate_token_on_enter_failure(self):
+ c = BrightDataClient(
+ token="tok_1234567890",
+ validate_token=True,
+ auto_create_zones=False,
+ )
+ with patch.object(c, "test_connection", new_callable=AsyncMock, return_value=False):
+ with pytest.raises(AuthenticationError, match="Token validation failed"):
+ async with c:
+ pass
+
+
+# ---------------------------------------------------------------------------
+# test_connection
+# ---------------------------------------------------------------------------
+
+
+class TestTestConnection:
+ @pytest.mark.asyncio
+ async def test_returns_true_on_200(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=[]))
+ )
+ result = await c.test_connection()
+ assert result is True
+ assert c._is_connected is True
+
+ @pytest.mark.asyncio
+ async def test_returns_false_on_401(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(return_value=MockContextManager(MockResponse(401)))
+ result = await c.test_connection()
+ assert result is False
+ assert c._is_connected is False
+
+ @pytest.mark.asyncio
+ async def test_returns_false_on_exception(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(side_effect=OSError("Network down"))
+ result = await c.test_connection()
+ assert result is False
+
+
+# ---------------------------------------------------------------------------
+# get_account_info
+# ---------------------------------------------------------------------------
+
+
+class TestGetAccountInfo:
+ @pytest.mark.asyncio
+ async def test_returns_account_info(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ zones = [{"name": "zone1"}, {"name": "zone2"}]
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=zones))
+ )
+ info = await c.get_account_info()
+
+ assert info["zone_count"] == 2
+ assert info["token_valid"] is True
+ assert len(info["zones"]) == 2
+
+ @pytest.mark.asyncio
+ async def test_caches_result(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=[{"name": "z"}]))
+ )
+ info1 = await c.get_account_info()
+ info2 = await c.get_account_info()
+
+ assert info1 is info2
+ # Should only call API once
+ c.engine.get_from_url.assert_called_once()
+
+ @pytest.mark.asyncio
+ async def test_refresh_bypasses_cache(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=[]))
+ )
+ await c.get_account_info()
+ await c.get_account_info(refresh=True)
+
+ assert c.engine.get_from_url.call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_401_raises_auth_error(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(401, text_data="Unauthorized"))
+ )
+ with pytest.raises(AuthenticationError, match="Invalid token"):
+ await c.get_account_info()
+
+ @pytest.mark.asyncio
+ async def test_500_raises_api_error(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(500, text_data="Server Error"))
+ )
+ with pytest.raises(APIError, match="Failed to get account info"):
+ await c.get_account_info()
+
+ @pytest.mark.asyncio
+ async def test_empty_zones_warns(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c.engine.get_from_url = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=[]))
+ )
+ with pytest.warns(UserWarning, match="No active zones"):
+ await c.get_account_info()
+
+
+# ---------------------------------------------------------------------------
+# list_zones / delete_zone
+# ---------------------------------------------------------------------------
+
+
+class TestZoneOperations:
+ @pytest.mark.asyncio
+ async def test_list_zones_delegates_to_zone_manager(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c._zone_manager = AsyncMock()
+ c._zone_manager.list_zones = AsyncMock(return_value=[{"name": "z1"}])
+
+ zones = await c.list_zones()
+
+ assert zones == [{"name": "z1"}]
+
+ @pytest.mark.asyncio
+ async def test_list_zones_creates_zone_manager(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ # Mock engine to avoid real HTTP
+ c.engine.get = MagicMock(
+ return_value=MockContextManager(MockResponse(200, json_data=[]))
+ )
+ zones = await c.list_zones()
+ assert c._zone_manager is not None
+
+ @pytest.mark.asyncio
+ async def test_delete_zone_delegates(self):
+ c = BrightDataClient(token="tok_1234567890", auto_create_zones=False)
+ async with c:
+ c._zone_manager = AsyncMock()
+ c._zone_manager.delete_zone = AsyncMock()
+
+ await c.delete_zone("test_zone")
+
+ c._zone_manager.delete_zone.assert_called_once_with("test_zone")
+
+
+# ---------------------------------------------------------------------------
+# __repr__
+# ---------------------------------------------------------------------------
+
+
+class TestRepr:
+ def test_includes_token_preview(self):
+ c = BrightDataClient(token="tok_1234567890_abcde")
+ r = repr(c)
+ assert "tok_12345" in r
+ assert "abcde" in r
+
+ def test_shows_not_tested_by_default(self):
+ c = BrightDataClient(token="tok_1234567890")
+ assert "Not tested" in repr(c)
diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py
deleted file mode 100644
index 4882828..0000000
--- a/tests/unit/test_constants.py
+++ /dev/null
@@ -1,274 +0,0 @@
-"""Unit tests for constants module."""
-
-from brightdata import constants
-
-
-class TestPollingConstants:
- """Test polling configuration constants."""
-
- def test_default_poll_interval_exists(self):
- """Test DEFAULT_POLL_INTERVAL constant exists."""
- assert hasattr(constants, "DEFAULT_POLL_INTERVAL")
-
- def test_default_poll_interval_is_integer(self):
- """Test DEFAULT_POLL_INTERVAL is an integer."""
- assert isinstance(constants.DEFAULT_POLL_INTERVAL, int)
-
- def test_default_poll_interval_is_positive(self):
- """Test DEFAULT_POLL_INTERVAL is positive."""
- assert constants.DEFAULT_POLL_INTERVAL > 0
-
- def test_default_poll_interval_value(self):
- """Test DEFAULT_POLL_INTERVAL has expected value."""
- assert constants.DEFAULT_POLL_INTERVAL == 10
-
- def test_default_poll_timeout_exists(self):
- """Test DEFAULT_POLL_TIMEOUT constant exists."""
- assert hasattr(constants, "DEFAULT_POLL_TIMEOUT")
-
- def test_default_poll_timeout_is_integer(self):
- """Test DEFAULT_POLL_TIMEOUT is an integer."""
- assert isinstance(constants.DEFAULT_POLL_TIMEOUT, int)
-
- def test_default_poll_timeout_is_positive(self):
- """Test DEFAULT_POLL_TIMEOUT is positive."""
- assert constants.DEFAULT_POLL_TIMEOUT > 0
-
- def test_default_poll_timeout_value(self):
- """Test DEFAULT_POLL_TIMEOUT has expected value."""
- assert constants.DEFAULT_POLL_TIMEOUT == 600
-
- def test_poll_timeout_greater_than_interval(self):
- """Test DEFAULT_POLL_TIMEOUT is greater than DEFAULT_POLL_INTERVAL."""
- assert constants.DEFAULT_POLL_TIMEOUT > constants.DEFAULT_POLL_INTERVAL
-
-
-class TestTimeoutConstants:
- """Test timeout configuration constants."""
-
- def test_default_timeout_short_exists(self):
- """Test DEFAULT_TIMEOUT_SHORT constant exists."""
- assert hasattr(constants, "DEFAULT_TIMEOUT_SHORT")
-
- def test_default_timeout_short_is_integer(self):
- """Test DEFAULT_TIMEOUT_SHORT is an integer."""
- assert isinstance(constants.DEFAULT_TIMEOUT_SHORT, int)
-
- def test_default_timeout_short_is_positive(self):
- """Test DEFAULT_TIMEOUT_SHORT is positive."""
- assert constants.DEFAULT_TIMEOUT_SHORT > 0
-
- def test_default_timeout_short_value(self):
- """Test DEFAULT_TIMEOUT_SHORT has expected value."""
- assert constants.DEFAULT_TIMEOUT_SHORT == 180
-
- def test_default_timeout_medium_exists(self):
- """Test DEFAULT_TIMEOUT_MEDIUM constant exists."""
- assert hasattr(constants, "DEFAULT_TIMEOUT_MEDIUM")
-
- def test_default_timeout_medium_is_integer(self):
- """Test DEFAULT_TIMEOUT_MEDIUM is an integer."""
- assert isinstance(constants.DEFAULT_TIMEOUT_MEDIUM, int)
-
- def test_default_timeout_medium_is_positive(self):
- """Test DEFAULT_TIMEOUT_MEDIUM is positive."""
- assert constants.DEFAULT_TIMEOUT_MEDIUM > 0
-
- def test_default_timeout_medium_value(self):
- """Test DEFAULT_TIMEOUT_MEDIUM has expected value."""
- assert constants.DEFAULT_TIMEOUT_MEDIUM == 240
-
- def test_default_timeout_long_exists(self):
- """Test DEFAULT_TIMEOUT_LONG constant exists."""
- assert hasattr(constants, "DEFAULT_TIMEOUT_LONG")
-
- def test_default_timeout_long_is_integer(self):
- """Test DEFAULT_TIMEOUT_LONG is an integer."""
- assert isinstance(constants.DEFAULT_TIMEOUT_LONG, int)
-
- def test_default_timeout_long_is_positive(self):
- """Test DEFAULT_TIMEOUT_LONG is positive."""
- assert constants.DEFAULT_TIMEOUT_LONG > 0
-
- def test_default_timeout_long_value(self):
- """Test DEFAULT_TIMEOUT_LONG has expected value."""
- assert constants.DEFAULT_TIMEOUT_LONG == 120
-
- def test_timeout_relationships(self):
- """Test timeout constants have logical relationships."""
- # Medium should be greater than short
- assert constants.DEFAULT_TIMEOUT_MEDIUM > constants.DEFAULT_TIMEOUT_SHORT
-
-
-class TestScraperConstants:
- """Test scraper configuration constants."""
-
- def test_default_min_poll_timeout_exists(self):
- """Test DEFAULT_MIN_POLL_TIMEOUT constant exists."""
- assert hasattr(constants, "DEFAULT_MIN_POLL_TIMEOUT")
-
- def test_default_min_poll_timeout_is_integer(self):
- """Test DEFAULT_MIN_POLL_TIMEOUT is an integer."""
- assert isinstance(constants.DEFAULT_MIN_POLL_TIMEOUT, int)
-
- def test_default_min_poll_timeout_is_positive(self):
- """Test DEFAULT_MIN_POLL_TIMEOUT is positive."""
- assert constants.DEFAULT_MIN_POLL_TIMEOUT > 0
-
- def test_default_min_poll_timeout_value(self):
- """Test DEFAULT_MIN_POLL_TIMEOUT has expected value."""
- assert constants.DEFAULT_MIN_POLL_TIMEOUT == 180
-
- def test_default_cost_per_record_exists(self):
- """Test DEFAULT_COST_PER_RECORD constant exists."""
- assert hasattr(constants, "DEFAULT_COST_PER_RECORD")
-
- def test_default_cost_per_record_is_float(self):
- """Test DEFAULT_COST_PER_RECORD is a float."""
- assert isinstance(constants.DEFAULT_COST_PER_RECORD, float)
-
- def test_default_cost_per_record_is_positive(self):
- """Test DEFAULT_COST_PER_RECORD is positive."""
- assert constants.DEFAULT_COST_PER_RECORD > 0
-
- def test_default_cost_per_record_value(self):
- """Test DEFAULT_COST_PER_RECORD has expected value."""
- assert constants.DEFAULT_COST_PER_RECORD == 0.001
-
-
-class TestConstantsDocumentation:
- """Test constants have proper documentation."""
-
- def test_default_poll_interval_has_docstring(self):
- """Test DEFAULT_POLL_INTERVAL has documentation."""
- # Check module docstrings or comments exist
- import inspect
-
- source = inspect.getsource(constants)
- assert "DEFAULT_POLL_INTERVAL" in source
-
- def test_constants_module_has_docstring(self):
- """Test constants module has docstring."""
- assert constants.__doc__ is not None
- assert len(constants.__doc__) > 0
-
-
-class TestConstantsUsage:
- """Test constants are used throughout the codebase."""
-
- def test_constants_imported_in_base_scraper(self):
- """Test constants are imported in base scraper."""
- from brightdata.scrapers import base
-
- # Should import from constants module
- import inspect
-
- source = inspect.getsource(base)
- assert "from ..constants import" in source or "constants" in source
-
- def test_constants_imported_in_polling(self):
- """Test constants are imported in polling utilities."""
- from brightdata.utils import polling
-
- import inspect
-
- source = inspect.getsource(polling)
- assert "from ..constants import" in source or "constants" in source
-
- def test_default_poll_interval_used_in_polling(self):
- """Test DEFAULT_POLL_INTERVAL is used in polling module."""
- from brightdata.utils import polling
-
- import inspect
-
- source = inspect.getsource(polling)
- assert "DEFAULT_POLL_INTERVAL" in source
-
-
-class TestConstantsImmutability:
- """Test constants maintain their values."""
-
- def test_constants_are_not_none(self):
- """Test all constants are not None."""
- assert constants.DEFAULT_POLL_INTERVAL is not None
- assert constants.DEFAULT_POLL_TIMEOUT is not None
- assert constants.DEFAULT_TIMEOUT_SHORT is not None
- assert constants.DEFAULT_TIMEOUT_MEDIUM is not None
- assert constants.DEFAULT_TIMEOUT_LONG is not None
- assert constants.DEFAULT_MIN_POLL_TIMEOUT is not None
- assert constants.DEFAULT_COST_PER_RECORD is not None
-
- def test_constants_have_expected_types(self):
- """Test all constants have expected types."""
- # Integer constants
- assert isinstance(constants.DEFAULT_POLL_INTERVAL, int)
- assert isinstance(constants.DEFAULT_POLL_TIMEOUT, int)
- assert isinstance(constants.DEFAULT_TIMEOUT_SHORT, int)
- assert isinstance(constants.DEFAULT_TIMEOUT_MEDIUM, int)
- assert isinstance(constants.DEFAULT_TIMEOUT_LONG, int)
- assert isinstance(constants.DEFAULT_MIN_POLL_TIMEOUT, int)
-
- # Float constant
- assert isinstance(constants.DEFAULT_COST_PER_RECORD, float)
-
-
-class TestConstantsExports:
- """Test constants module exports."""
-
- def test_can_import_constants_from_brightdata(self):
- """Test can import constants from brightdata package."""
- from brightdata import constants as const
-
- assert const is not None
- assert hasattr(const, "DEFAULT_POLL_INTERVAL")
-
- def test_can_import_specific_constants(self):
- """Test can import specific constants."""
- from brightdata.constants import (
- DEFAULT_POLL_INTERVAL,
- DEFAULT_POLL_TIMEOUT,
- DEFAULT_TIMEOUT_SHORT,
- DEFAULT_TIMEOUT_MEDIUM,
- DEFAULT_TIMEOUT_LONG,
- DEFAULT_MIN_POLL_TIMEOUT,
- DEFAULT_COST_PER_RECORD,
- )
-
- assert DEFAULT_POLL_INTERVAL is not None
- assert DEFAULT_POLL_TIMEOUT is not None
- assert DEFAULT_TIMEOUT_SHORT is not None
- assert DEFAULT_TIMEOUT_MEDIUM is not None
- assert DEFAULT_TIMEOUT_LONG is not None
- assert DEFAULT_MIN_POLL_TIMEOUT is not None
- assert DEFAULT_COST_PER_RECORD is not None
-
-
-class TestConstantsReasonableValues:
- """Test constants have reasonable values for production use."""
-
- def test_poll_interval_is_reasonable(self):
- """Test poll interval is reasonable (not too frequent, not too slow)."""
- # Should be between 1 and 60 seconds
- assert 1 <= constants.DEFAULT_POLL_INTERVAL <= 60
-
- def test_poll_timeout_is_reasonable(self):
- """Test poll timeout is reasonable."""
- # Should be at least 1 minute, but not more than 30 minutes
- assert 60 <= constants.DEFAULT_POLL_TIMEOUT <= 1800
-
- def test_timeouts_are_reasonable(self):
- """Test all timeout values are reasonable for API operations."""
- # All timeouts should be between 30 seconds and 10 minutes
- assert 30 <= constants.DEFAULT_TIMEOUT_SHORT <= 600
- assert 30 <= constants.DEFAULT_TIMEOUT_MEDIUM <= 600
- assert 30 <= constants.DEFAULT_TIMEOUT_LONG <= 600
-
- def test_cost_per_record_is_reasonable(self):
- """Test cost per record is reasonable."""
- # Should be between $0.0001 and $0.01 per record
- assert 0.0001 <= constants.DEFAULT_COST_PER_RECORD <= 0.01
-
- def test_min_poll_timeout_is_reasonable(self):
- """Test minimum poll timeout is reasonable."""
- # Should be at least 1 minute
- assert constants.DEFAULT_MIN_POLL_TIMEOUT >= 60
diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py
index 958f4b2..2b82641 100644
--- a/tests/unit/test_engine.py
+++ b/tests/unit/test_engine.py
@@ -1 +1,314 @@
-"""Unit tests for engine."""
+"""Tests for core/engine.py — AsyncEngine HTTP communication."""
+
+import asyncio
+import ssl
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import aiohttp
+import pytest
+
+from brightdata.core.engine import AsyncEngine
+from brightdata.exceptions import AuthenticationError, NetworkError, SSLError
+
+
+# ---------------------------------------------------------------------------
+# Initialization
+# ---------------------------------------------------------------------------
+
+
+class TestEngineInit:
+ def test_stores_bearer_token(self):
+ engine = AsyncEngine(bearer_token="tok_abc")
+ assert engine.bearer_token == "tok_abc"
+
+ def test_default_timeout(self):
+ engine = AsyncEngine(bearer_token="tok")
+ assert engine.timeout.total == 30
+
+ def test_custom_timeout(self):
+ engine = AsyncEngine(bearer_token="tok", timeout=120)
+ assert engine.timeout.total == 120
+
+ def test_default_rate_limit(self):
+ engine = AsyncEngine(bearer_token="tok")
+ assert engine._rate_limit == AsyncEngine.DEFAULT_RATE_LIMIT
+
+ def test_custom_rate_limit(self):
+ engine = AsyncEngine(bearer_token="tok", rate_limit=5, rate_period=2.0)
+ assert engine._rate_limit == 5
+ assert engine._rate_period == 2.0
+
+ def test_session_none_before_enter(self):
+ engine = AsyncEngine(bearer_token="tok")
+ assert engine._session is None
+
+
+# ---------------------------------------------------------------------------
+# Context manager
+# ---------------------------------------------------------------------------
+
+
+class TestEngineContextManager:
+ @pytest.mark.asyncio
+ async def test_creates_session_on_enter(self):
+ engine = AsyncEngine(bearer_token="tok_abc")
+ async with engine:
+ assert engine._session is not None
+ assert not engine._session.closed
+
+ @pytest.mark.asyncio
+ async def test_closes_session_on_exit(self):
+ engine = AsyncEngine(bearer_token="tok_abc")
+ async with engine:
+ session = engine._session
+ assert engine._session is None
+ assert session.closed
+
+ @pytest.mark.asyncio
+ async def test_idempotent_enter(self):
+ """Calling __aenter__ twice should reuse the same session."""
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ session1 = engine._session
+ await engine.__aenter__()
+ session2 = engine._session
+ assert session1 is session2
+
+ @pytest.mark.asyncio
+ async def test_session_headers_contain_auth(self):
+ engine = AsyncEngine(bearer_token="my_secret_token")
+ async with engine:
+ headers = engine._session.headers
+ assert headers["Authorization"] == "Bearer my_secret_token"
+ assert headers["Content-Type"] == "application/json"
+ assert "brightdata-sdk/" in headers["User-Agent"]
+
+
+# ---------------------------------------------------------------------------
+# Request routing (without real HTTP — tests the method dispatch)
+# ---------------------------------------------------------------------------
+
+
+class TestRequestRouting:
+ @pytest.mark.asyncio
+ async def test_raises_if_no_session(self):
+ engine = AsyncEngine(bearer_token="tok")
+ with pytest.raises(RuntimeError, match="must be used as async context manager"):
+ engine.request("GET", "/test")
+
+ @pytest.mark.asyncio
+ async def test_get_delegates_to_request(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ with patch.object(engine, "request", return_value="cm") as mock_req:
+ result = engine.get("/endpoint", params={"a": "1"})
+ mock_req.assert_called_once_with(
+ "GET", "/endpoint", params={"a": "1"}, headers=None
+ )
+
+ @pytest.mark.asyncio
+ async def test_post_delegates_to_request(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ with patch.object(engine, "request", return_value="cm") as mock_req:
+ result = engine.post("/endpoint", json_data={"k": "v"})
+ mock_req.assert_called_once_with(
+ "POST", "/endpoint", json_data={"k": "v"}, params=None, headers=None
+ )
+
+ @pytest.mark.asyncio
+ async def test_delete_delegates_to_request(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ with patch.object(engine, "request", return_value="cm") as mock_req:
+ result = engine.delete("/endpoint")
+ mock_req.assert_called_once_with(
+ "DELETE", "/endpoint", json_data=None, params=None, headers=None
+ )
+
+ @pytest.mark.asyncio
+ async def test_request_builds_full_url(self):
+ """request() should prepend BASE_URL to the endpoint."""
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ cm = engine.request("GET", "/v3/test")
+ # The URL is stored inside the ResponseContextManager
+ assert cm._url == f"{AsyncEngine.BASE_URL}/v3/test"
+
+ @pytest.mark.asyncio
+ async def test_request_merges_custom_headers(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ cm = engine.request("GET", "/test", headers={"X-Custom": "val"})
+ assert cm._headers["X-Custom"] == "val"
+ # Original auth header should still be there
+ assert "Bearer tok" in cm._headers["Authorization"]
+
+ @pytest.mark.asyncio
+ async def test_post_to_url_uses_full_url(self):
+ """post_to_url should NOT prepend BASE_URL."""
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ cm = engine.post_to_url("https://custom.api.com/trigger", json_data={"x": 1})
+ assert cm._url == "https://custom.api.com/trigger"
+
+ @pytest.mark.asyncio
+ async def test_get_from_url_uses_full_url(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ cm = engine.get_from_url("https://custom.api.com/data", params={"fmt": "json"})
+ assert cm._url == "https://custom.api.com/data"
+ assert cm._params == {"fmt": "json"}
+
+
+# ---------------------------------------------------------------------------
+# Error handling (mock aiohttp session to simulate failures)
+# ---------------------------------------------------------------------------
+
+
+class TestErrorHandling:
+ @pytest.mark.asyncio
+ async def test_401_raises_authentication_error(self):
+ engine = AsyncEngine(bearer_token="bad_token")
+ async with engine:
+ # Mock the session.request to return a 401 response
+ mock_resp = AsyncMock()
+ mock_resp.status = 401
+ mock_resp.text = AsyncMock(return_value="Unauthorized")
+ mock_resp.release = AsyncMock()
+ engine._session.request = AsyncMock(return_value=mock_resp)
+
+ cm = engine.get("/test")
+ with pytest.raises(AuthenticationError, match="Unauthorized"):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_403_raises_authentication_error(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ mock_resp = AsyncMock()
+ mock_resp.status = 403
+ mock_resp.text = AsyncMock(return_value="Forbidden")
+ mock_resp.release = AsyncMock()
+ engine._session.request = AsyncMock(return_value=mock_resp)
+
+ cm = engine.get("/test")
+ with pytest.raises(AuthenticationError, match="Forbidden"):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_200_returns_response(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ mock_resp = AsyncMock()
+ mock_resp.status = 200
+ mock_resp.json = AsyncMock(return_value={"ok": True})
+ mock_resp.close = MagicMock()
+ engine._session.request = AsyncMock(return_value=mock_resp)
+
+ async with engine.get("/test") as resp:
+ assert resp.status == 200
+ data = await resp.json()
+ assert data == {"ok": True}
+
+ @pytest.mark.asyncio
+ async def test_network_error_raises_network_error(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ engine._session.request = AsyncMock(
+ side_effect=aiohttp.ClientError("Connection refused")
+ )
+
+ cm = engine.get("/test")
+ with pytest.raises(NetworkError, match="Network error"):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_timeout_raises_timeout_error(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ engine._session.request = AsyncMock(side_effect=asyncio.TimeoutError())
+
+ cm = engine.get("/test")
+ with pytest.raises(TimeoutError, match="timeout"):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_ssl_error_raises_ssl_error(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ ssl_err = aiohttp.ClientConnectorCertificateError(
+ connection_key=MagicMock(),
+ certificate_error=ssl.SSLCertVerificationError("CERTIFICATE_VERIFY_FAILED"),
+ )
+ engine._session.request = AsyncMock(side_effect=ssl_err)
+
+ cm = engine.get("/test")
+ with pytest.raises(SSLError):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_os_error_with_ssl_raises_ssl_error(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ engine._session.request = AsyncMock(
+ side_effect=OSError("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed")
+ )
+
+ cm = engine.get("/test")
+ with pytest.raises(SSLError):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_generic_os_error_raises_network_error(self):
+ """Non-SSL OSError should be NetworkError, not SSLError."""
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ engine._session.request = AsyncMock(side_effect=OSError("Connection reset by peer"))
+
+ cm = engine.get("/test")
+ with pytest.raises(NetworkError, match="Network error"):
+ async with cm:
+ pass
+
+ @pytest.mark.asyncio
+ async def test_response_closed_on_exit(self):
+ engine = AsyncEngine(bearer_token="tok")
+ async with engine:
+ mock_resp = MagicMock()
+ mock_resp.status = 200
+ mock_resp.close = MagicMock()
+ engine._session.request = AsyncMock(return_value=mock_resp)
+
+ async with engine.get("/test") as resp:
+ pass
+ mock_resp.close.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# Rate limiter
+# ---------------------------------------------------------------------------
+
+
+class TestRateLimiter:
+ @pytest.mark.asyncio
+ async def test_rate_limiter_created_on_enter(self):
+ engine = AsyncEngine(bearer_token="tok", rate_limit=5)
+ async with engine:
+ if engine._rate_limiter is not None:
+ # aiolimiter installed
+ assert engine._rate_limiter is not None
+ # If aiolimiter not installed, limiter will be None — that's OK
+
+ @pytest.mark.asyncio
+ async def test_rate_limiter_cleared_on_exit(self):
+ engine = AsyncEngine(bearer_token="tok", rate_limit=5)
+ async with engine:
+ pass
+ assert engine._rate_limiter is None
diff --git a/tests/unit/test_engine_sharing.py b/tests/unit/test_engine_sharing.py
deleted file mode 100644
index 4aa6ccd..0000000
--- a/tests/unit/test_engine_sharing.py
+++ /dev/null
@@ -1,217 +0,0 @@
-"""
-Test script to verify AsyncEngine sharing across scrapers.
-
-This script verifies that the AsyncEngine duplication fix works correctly by:
-1. Counting AsyncEngine instances before/after creating client
-2. Accessing multiple scrapers and verifying only one engine exists
-3. Ensuring resource efficiency and proper engine reuse
-
-Expected output:
-- Before creating client: 0 engines
-- After creating client: 1 engine
-- After accessing all scrapers: 1 engine (SHOULD STILL BE 1)
-
-If this test passes, the fix is working correctly!
-"""
-
-import gc
-import sys
-import os
-
-# Add src to path so we can import brightdata
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
-
-from brightdata import BrightDataClient
-from brightdata.core.engine import AsyncEngine
-
-
-def count_engines():
- """Count the number of AsyncEngine instances in memory."""
- gc.collect() # Force garbage collection to get accurate count
- engines = [obj for obj in gc.get_objects() if isinstance(obj, AsyncEngine)]
- return len(engines)
-
-
-def test_engine_sharing():
- """Test that only one engine is created and shared across all scrapers."""
-
- print("=" * 70)
- print("AsyncEngine Sharing Test")
- print("=" * 70)
- print()
-
- # Step 1: Check baseline (should be 0)
- initial_count = count_engines()
- print(f"✓ Step 1: Before creating client: {initial_count} engine(s)")
-
- if initial_count != 0:
- print(f" ⚠️ Warning: Expected 0 engines, found {initial_count}")
- print()
-
- # Step 2: Create client (should create 1 engine)
- print("✓ Step 2: Creating BrightDataClient...")
-
- # Try to load token from environment, or use placeholder
- token = os.getenv("BRIGHTDATA_API_TOKEN")
- if not token:
- print(" ⚠️ Warning: No BRIGHTDATA_API_TOKEN found, using placeholder")
- token = "test_token_placeholder_12345"
-
- client = BrightDataClient(token=token)
-
- after_client_count = count_engines()
- print(f"✓ Step 3: After creating client: {after_client_count} engine(s)")
-
- if after_client_count != 1:
- print(f" ❌ FAILED: Expected 1 engine, found {after_client_count}")
- return False
- print()
-
- # Step 3: Access all scrapers (should still be 1 engine)
- print("✓ Step 4: Accessing all scrapers...")
-
- scrapers_accessed = []
-
- try:
- # Access scrape services
- _ = client.scrape.amazon
- scrapers_accessed.append("amazon")
-
- _ = client.scrape.linkedin
- scrapers_accessed.append("linkedin")
-
- _ = client.scrape.facebook
- scrapers_accessed.append("facebook")
-
- _ = client.scrape.instagram
- scrapers_accessed.append("instagram")
-
- _ = client.scrape.chatgpt
- scrapers_accessed.append("chatgpt")
-
- # Access search services
- _ = client.search.linkedin
- scrapers_accessed.append("search.linkedin")
-
- _ = client.search.instagram
- scrapers_accessed.append("search.instagram")
-
- _ = client.search.chatGPT
- scrapers_accessed.append("search.chatGPT")
-
- print(f" Accessed {len(scrapers_accessed)} scrapers: {', '.join(scrapers_accessed)}")
-
- except Exception as e:
- print(f" ⚠️ Warning: Error accessing scrapers: {e}")
-
- print()
-
- # Step 4: Count engines after accessing all scrapers
- after_scrapers_count = count_engines()
- print(f"✓ Step 5: After accessing all scrapers: {after_scrapers_count} engine(s)")
- print()
-
- # Verify the result
- print("=" * 70)
- print("Test Results")
- print("=" * 70)
-
- if after_scrapers_count == 1:
- print("✅ SUCCESS! Only 1 AsyncEngine instance exists.")
- print(" All scrapers are sharing the client's engine.")
- print(" Resource efficiency: OPTIMAL")
- print()
- print(" Benefits:")
- print(" • Single HTTP connection pool")
- print(" • Unified rate limiting")
- print(" • Reduced memory usage")
- print(" • Better connection reuse")
- return True
- else:
- print(f"❌ FAILED! Found {after_scrapers_count} AsyncEngine instances.")
- print(" Expected: 1 engine (shared across all scrapers)")
- print(f" Actual: {after_scrapers_count} engines (resource duplication)")
- print()
- print(" This means:")
- print(" • Multiple connection pools created")
- print(" • Inefficient resource usage")
- print(" • Engine duplication not fixed")
- return False
-
-
-def test_standalone_scraper():
- """Test that standalone scrapers still work (backwards compatibility)."""
-
- print()
- print("=" * 70)
- print("Standalone Scraper Test (Backwards Compatibility)")
- print("=" * 70)
- print()
-
- # Clear any existing engines
- gc.collect()
- initial_count = count_engines()
-
- print(f"✓ Initial engine count: {initial_count}")
-
- # Import and create a standalone scraper
- from brightdata.scrapers.amazon import AmazonScraper
-
- print("✓ Creating standalone AmazonScraper (without passing engine)...")
-
- try:
- token = os.getenv("BRIGHTDATA_API_TOKEN", "test_token_placeholder_12345")
- AmazonScraper(bearer_token=token)
-
- standalone_count = count_engines()
- print(f"✓ After creating standalone scraper: {standalone_count} engine(s)")
-
- expected_count = initial_count + 1
- if standalone_count == expected_count:
- print("✅ SUCCESS! Standalone scraper creates its own engine.")
- print(" Backwards compatibility: MAINTAINED")
- return True
- else:
- print(f"❌ FAILED! Expected {expected_count} engines, found {standalone_count}")
- return False
-
- except Exception as e:
- print(f"⚠️ Warning: Could not create standalone scraper: {e}")
- print(" (This is expected if bearer token is missing)")
- return True # Don't fail the test if token is missing
-
-
-if __name__ == "__main__":
- print()
- print("╔" + "═" * 68 + "╗")
- print("║" + " " * 15 + "AsyncEngine Duplication Fix Test" + " " * 20 + "║")
- print("╚" + "═" * 68 + "╝")
- print()
-
- # Run both tests
- test1_passed = test_engine_sharing()
- test2_passed = test_standalone_scraper()
-
- print()
- print("=" * 70)
- print("Final Results")
- print("=" * 70)
- print()
-
- if test1_passed and test2_passed:
- print("✅ ALL TESTS PASSED!")
- print()
- print("The AsyncEngine duplication fix is working correctly:")
- print("• Single engine shared across all client scrapers ✓")
- print("• Standalone scrapers still create their own engine ✓")
- print("• Backwards compatibility maintained ✓")
- print("• Resource efficiency achieved ✓")
- sys.exit(0)
- else:
- print("❌ SOME TESTS FAILED")
- print()
- if not test1_passed:
- print("• Engine sharing test failed - duplication still exists")
- if not test2_passed:
- print("• Standalone scraper test failed - backwards compatibility broken")
- sys.exit(1)
diff --git a/tests/unit/test_facebook.py b/tests/unit/test_facebook.py
deleted file mode 100644
index ed2bfa3..0000000
--- a/tests/unit/test_facebook.py
+++ /dev/null
@@ -1,262 +0,0 @@
-"""Unit tests for Facebook scraper."""
-
-from brightdata import BrightDataClient
-from brightdata.scrapers.facebook import FacebookScraper
-
-
-class TestFacebookScraperURLBased:
- """Test Facebook scraper (URL-based extraction)."""
-
- def test_facebook_scraper_has_posts_by_profile_method(self):
- """Test Facebook scraper has posts_by_profile method (async-first API)."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts_by_profile")
- assert callable(scraper.posts_by_profile)
-
- def test_facebook_scraper_has_posts_by_group_method(self):
- """Test Facebook scraper has posts_by_group method (async-first API)."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts_by_group")
- assert callable(scraper.posts_by_group)
-
- def test_facebook_scraper_has_posts_by_url_method(self):
- """Test Facebook scraper has posts_by_url method (async-first API)."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts_by_url")
- assert callable(scraper.posts_by_url)
-
- def test_facebook_scraper_has_comments_method(self):
- """Test Facebook scraper has comments method (async-first API)."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "comments")
- assert callable(scraper.comments)
-
- def test_facebook_scraper_has_reels_method(self):
- """Test Facebook scraper has reels method (async-first API)."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reels")
- assert callable(scraper.reels)
-
- def test_posts_by_profile_method_signature(self):
- """Test posts_by_profile method has correct signature."""
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts_by_profile)
-
- # Required: url parameter
- assert "url" in sig.parameters
-
- # Optional filters
- assert "num_of_posts" in sig.parameters
- assert "posts_to_not_include" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 240
-
- def test_posts_by_group_method_signature(self):
- """Test posts_by_group method has correct signature."""
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts_by_group)
-
- # Required: url
- assert "url" in sig.parameters
-
- # Optional filters
- assert "num_of_posts" in sig.parameters
- assert "posts_to_not_include" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 240
-
- def test_posts_by_url_method_signature(self):
- """Test posts_by_url method has correct signature."""
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts_by_url)
-
- assert "url" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 240
-
- def test_comments_method_signature(self):
- """Test comments method has correct signature."""
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.comments)
-
- assert "url" in sig.parameters
- assert "num_of_comments" in sig.parameters
- assert "comments_to_not_include" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 240
-
- def test_reels_method_signature(self):
- """Test reels method has correct signature."""
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reels)
-
- assert "url" in sig.parameters
- assert "num_of_posts" in sig.parameters
- assert "posts_to_not_include" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 240
-
-
-class TestFacebookDatasetIDs:
- """Test Facebook has correct dataset IDs."""
-
- def test_scraper_has_all_dataset_ids(self):
- """Test scraper has dataset IDs for all types."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert scraper.DATASET_ID # Default: Posts by Profile
- assert scraper.DATASET_ID_POSTS_PROFILE
- assert scraper.DATASET_ID_POSTS_GROUP
- assert scraper.DATASET_ID_POSTS_URL
- assert scraper.DATASET_ID_COMMENTS
- assert scraper.DATASET_ID_REELS
-
- # All should start with gd_
- assert scraper.DATASET_ID.startswith("gd_")
- assert scraper.DATASET_ID_POSTS_PROFILE.startswith("gd_")
- assert scraper.DATASET_ID_POSTS_GROUP.startswith("gd_")
- assert scraper.DATASET_ID_POSTS_URL.startswith("gd_")
- assert scraper.DATASET_ID_COMMENTS.startswith("gd_")
- assert scraper.DATASET_ID_REELS.startswith("gd_")
-
- def test_scraper_has_platform_name(self):
- """Test scraper has correct platform name."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "facebook"
-
- def test_scraper_has_cost_per_record(self):
- """Test scraper has cost per record."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "COST_PER_RECORD")
- assert isinstance(scraper.COST_PER_RECORD, (int, float))
- assert scraper.COST_PER_RECORD > 0
-
-
-class TestFacebookScraperRegistration:
- """Test Facebook scraper is registered correctly."""
-
- def test_facebook_is_registered(self):
- """Test Facebook scraper is in registry."""
- from brightdata.scrapers.registry import is_platform_supported, get_registered_platforms
-
- assert is_platform_supported("facebook")
- assert "facebook" in get_registered_platforms()
-
- def test_can_get_facebook_scraper_from_registry(self):
- """Test can get Facebook scraper from registry."""
- from brightdata.scrapers.registry import get_scraper_for
-
- scraper_class = get_scraper_for("facebook")
- assert scraper_class is not None
- assert scraper_class.__name__ == "FacebookScraper"
-
-
-class TestFacebookClientIntegration:
- """Test Facebook scraper integration with BrightDataClient."""
-
- def test_client_has_facebook_scraper_access(self):
- """Test client provides access to Facebook scraper."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client, "scrape")
- assert hasattr(client.scrape, "facebook")
-
- def test_client_facebook_scraper_has_all_methods(self):
- """Test client.scrape.facebook has all Facebook methods."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.scrape.facebook, "posts_by_profile")
- assert hasattr(client.scrape.facebook, "posts_by_group")
- assert hasattr(client.scrape.facebook, "posts_by_url")
- assert hasattr(client.scrape.facebook, "comments")
- assert hasattr(client.scrape.facebook, "reels")
-
- def test_facebook_scraper_instance_from_client(self):
- """Test Facebook scraper instance is FacebookScraper."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert isinstance(client.scrape.facebook, FacebookScraper)
-
-
-class TestFacebookScraperConfiguration:
- """Test Facebook scraper configuration."""
-
- def test_scraper_initialization_with_token(self):
- """Test scraper can be initialized with bearer token."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert scraper.bearer_token == "test_token_123456789"
-
- def test_scraper_has_engine(self):
- """Test scraper has engine instance."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "engine")
- assert scraper.engine is not None
-
- def test_scraper_has_api_client(self):
- """Test scraper has API client."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "api_client")
- assert scraper.api_client is not None
-
- def test_scraper_has_workflow_executor(self):
- """Test scraper has workflow executor."""
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "workflow_executor")
- assert scraper.workflow_executor is not None
-
-
-class TestFacebookScraperExports:
- """Test Facebook scraper is properly exported."""
-
- def test_facebook_scraper_in_module_exports(self):
- """Test FacebookScraper is in scrapers module __all__."""
- from brightdata import scrapers
-
- assert "FacebookScraper" in scrapers.__all__
-
- def test_can_import_facebook_scraper_directly(self):
- """Test can import FacebookScraper directly."""
- from brightdata.scrapers import FacebookScraper as FB
-
- assert FB is not None
- assert FB.__name__ == "FacebookScraper"
-
- def test_can_import_from_facebook_submodule(self):
- """Test can import from facebook submodule."""
- from brightdata.scrapers.facebook import FacebookScraper as FB
-
- assert FB is not None
- assert FB.__name__ == "FacebookScraper"
diff --git a/tests/unit/test_function_detection.py b/tests/unit/test_function_detection.py
deleted file mode 100644
index fcf1319..0000000
--- a/tests/unit/test_function_detection.py
+++ /dev/null
@@ -1,251 +0,0 @@
-"""Unit tests for function detection utilities."""
-
-from brightdata.utils.function_detection import get_caller_function_name
-
-
-class TestFunctionDetection:
- """Test function name detection utilities."""
-
- def test_get_caller_function_name_exists(self):
- """Test get_caller_function_name function exists."""
- assert callable(get_caller_function_name)
-
- def test_get_caller_function_name_returns_string(self):
- """Test get_caller_function_name returns a string."""
-
- def test_function():
- return get_caller_function_name()
-
- result = test_function()
- assert isinstance(result, str)
-
- def test_get_caller_function_name_detects_caller(self):
- """Test get_caller_function_name detects calling function name."""
-
- def outer_function():
- return get_caller_function_name()
-
- result = outer_function()
- # Should detect 'outer_function' or similar
- assert len(result) > 0
-
- def test_get_caller_function_name_in_nested_calls(self):
- """Test get_caller_function_name works in nested function calls."""
-
- def level_3():
- return get_caller_function_name()
-
- def level_2():
- return level_3()
-
- def level_1():
- return level_2()
-
- result = level_1()
- # Should return a valid function name
- assert isinstance(result, str)
- assert len(result) > 0
-
- def test_get_caller_function_name_handles_no_caller(self):
- """Test get_caller_function_name handles cases with no clear caller."""
- # Call from module level (no function context)
- result = get_caller_function_name()
- # Should return something (empty string, None, or a default)
- assert result is not None
-
-
-class TestFunctionDetectionInScrapers:
- """Test function detection is used in scrapers."""
-
- def test_function_detection_imported_in_base_scraper(self):
- """Test function detection is imported in base scraper."""
- from brightdata.scrapers import base
-
- import inspect
-
- source = inspect.getsource(base)
- assert "get_caller_function_name" in source or "function_detection" in source
-
- def test_function_detection_used_for_sdk_function_parameter(self):
- """Test function detection is used to set sdk_function parameter."""
- from brightdata.scrapers import base
-
- # Check if sdk_function parameter is used in base scraper
- import inspect
-
- source = inspect.getsource(base)
- assert "sdk_function" in source
-
-
-class TestSDKFunctionParameterTracking:
- """Test sdk_function parameter tracking in scrapers."""
-
- def test_amazon_scraper_methods_accept_sdk_function(self):
- """Test Amazon scraper methods can track sdk_function."""
- from brightdata.scrapers.amazon import AmazonScraper
- import inspect
-
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- # Amazon uses _scrape_with_params which may have sdk_function
- # Note: Amazon's _scrape_urls doesn't have sdk_function, but it's
- # passed through workflow_executor.execute() which does accept it
- if hasattr(scraper, "_scrape_with_params"):
- inspect.signature(scraper._scrape_with_params)
- # sdk_function is handled internally via get_caller_function_name()
- assert True # Test passes - sdk_function is tracked via function detection
-
- def test_linkedin_scraper_methods_accept_sdk_function(self):
- """Test LinkedIn scraper methods can track sdk_function."""
- from brightdata.scrapers.linkedin import LinkedInScraper
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- # LinkedIn uses _scrape_with_params which may have sdk_function
- # Note: LinkedIn's _scrape_urls doesn't have sdk_function, but it's
- # passed through workflow_executor.execute() which does accept it
- if hasattr(scraper, "_scrape_with_params"):
- inspect.signature(scraper._scrape_with_params)
- # sdk_function is handled internally via get_caller_function_name()
- assert True # Test passes - sdk_function is tracked via function detection
-
- def test_facebook_scraper_methods_accept_sdk_function(self):
- """Test Facebook scraper methods can track sdk_function."""
- from brightdata.scrapers.facebook import FacebookScraper
- import inspect
-
- scraper = FacebookScraper(bearer_token="test_token_123456789")
-
- # Check if internal methods accept sdk_function parameter
- if hasattr(scraper, "_scrape_urls"):
- sig = inspect.signature(scraper._scrape_urls)
- assert "sdk_function" in sig.parameters
-
- def test_instagram_scraper_methods_use_function_detection(self):
- """Test Instagram scraper methods use function detection internally."""
- from brightdata.scrapers.instagram import InstagramScraper
- import inspect
-
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- # Instagram scraper's _scrape_urls calls get_caller_function_name() internally
- # rather than accepting sdk_function as a parameter
- if hasattr(scraper, "_scrape_urls"):
- # Verify the method exists and is callable
- assert callable(scraper._scrape_urls)
- # Check it has the expected parameters (url, dataset_id, timeout)
- sig = inspect.signature(scraper._scrape_urls)
- assert "url" in sig.parameters
- assert "dataset_id" in sig.parameters
- assert "timeout" in sig.parameters
-
-
-class TestSDKFunctionUsagePatterns:
- """Test sdk_function parameter usage patterns."""
-
- def test_sdk_function_can_be_none(self):
- """Test sdk_function parameter can be None."""
- # Function detection should handle None gracefully
- result = get_caller_function_name()
- # Should return a string (possibly empty) or None, not crash
- assert result is None or isinstance(result, str)
-
- def test_sdk_function_provides_context_for_monitoring(self):
- """Test sdk_function provides context for monitoring and analytics."""
- # This is a design test - sdk_function should be passed through
- # the workflow executor to enable analytics
- from brightdata.scrapers.workflow import WorkflowExecutor
- import inspect
-
- # Check if WorkflowExecutor.execute accepts sdk_function
- sig = inspect.signature(WorkflowExecutor.execute)
- assert "sdk_function" in sig.parameters
-
-
-class TestFunctionDetectionEdgeCases:
- """Test function detection edge cases."""
-
- def test_function_detection_with_lambda(self):
- """Test function detection with lambda functions."""
-
- def func():
- return get_caller_function_name()
-
- result = func()
- # Should handle lambda gracefully
- assert result is None or isinstance(result, str)
-
- def test_function_detection_with_method(self):
- """Test function detection with class methods."""
-
- class TestClass:
- def method(self):
- return get_caller_function_name()
-
- obj = TestClass()
- result = obj.method()
- # Should detect method name
- assert isinstance(result, str)
-
- def test_function_detection_with_static_method(self):
- """Test function detection with static methods."""
-
- class TestClass:
- @staticmethod
- def static_method():
- return get_caller_function_name()
-
- result = TestClass.static_method()
- # Should handle static method
- assert result is None or isinstance(result, str)
-
- def test_function_detection_with_class_method(self):
- """Test function detection with class methods."""
-
- class TestClass:
- @classmethod
- def class_method(cls):
- return get_caller_function_name()
-
- result = TestClass.class_method()
- # Should handle class method
- assert result is None or isinstance(result, str)
-
-
-class TestFunctionDetectionPerformance:
- """Test function detection performance characteristics."""
-
- def test_function_detection_is_fast(self):
- """Test function detection doesn't add significant overhead."""
- import time
-
- def test_function():
- return get_caller_function_name()
-
- # Measure time for 1000 calls
- start = time.time()
- for _ in range(1000):
- test_function()
- elapsed = time.time() - start
-
- # Should complete in less than 1 second for 1000 calls
- assert elapsed < 1.0
-
- def test_function_detection_doesnt_cause_memory_leak(self):
- """Test function detection doesn't cause memory leaks."""
- import sys
-
- def test_function():
- return get_caller_function_name()
-
- # Get initial reference count
- initial_refs = sys.getrefcount(test_function)
-
- # Call many times
- for _ in range(100):
- test_function()
-
- # Reference count shouldn't grow significantly
- final_refs = sys.getrefcount(test_function)
- assert final_refs <= initial_refs + 5 # Allow small variation
diff --git a/tests/unit/test_instagram.py b/tests/unit/test_instagram.py
deleted file mode 100644
index 0f43e64..0000000
--- a/tests/unit/test_instagram.py
+++ /dev/null
@@ -1,390 +0,0 @@
-"""Unit tests for Instagram scraper."""
-
-from brightdata import BrightDataClient
-from brightdata.scrapers.instagram import InstagramScraper, InstagramSearchScraper
-
-
-class TestInstagramScraperURLBased:
- """Test Instagram scraper (URL-based extraction)."""
-
- def test_instagram_scraper_has_profiles_method(self):
- """Test Instagram scraper has profiles method (async-first API)."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "profiles")
- assert callable(scraper.profiles)
-
- def test_instagram_scraper_has_posts_method(self):
- """Test Instagram scraper has posts method (async-first API)."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts")
- assert callable(scraper.posts)
-
- def test_instagram_scraper_has_comments_method(self):
- """Test Instagram scraper has comments method (async-first API)."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "comments")
- assert callable(scraper.comments)
-
- def test_instagram_scraper_has_reels_method(self):
- """Test Instagram scraper has reels method (async-first API)."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reels")
- assert callable(scraper.reels)
-
- def test_profiles_method_signature(self):
- """Test profiles method has correct signature."""
- import inspect
-
- scraper = InstagramScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.profiles)
-
- # Required: url parameter
- assert "url" in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults (180s = DEFAULT_TIMEOUT_SHORT, same as LinkedIn)
- assert sig.parameters["timeout"].default == 180
-
- def test_posts_method_signature(self):
- """Test posts method has correct signature."""
- import inspect
-
- scraper = InstagramScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts)
-
- assert "url" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_comments_method_signature(self):
- """Test comments method has correct signature."""
- import inspect
-
- scraper = InstagramScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.comments)
-
- assert "url" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_reels_method_signature(self):
- """Test reels method has correct signature."""
- import inspect
-
- scraper = InstagramScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reels)
-
- assert "url" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
-
-class TestInstagramSearchScraper:
- """Test Instagram search scraper (parameter-based discovery)."""
-
- def test_instagram_search_scraper_has_profiles_method(self):
- """Test Instagram search scraper has profiles method for username discovery."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "profiles")
- assert callable(scraper.profiles)
-
- def test_instagram_search_scraper_has_posts_method(self):
- """Test Instagram search scraper has posts method (async-first API)."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts")
- assert callable(scraper.posts)
-
- def test_instagram_search_scraper_has_reels_method(self):
- """Test Instagram search scraper has reels method (async-first API)."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reels")
- assert callable(scraper.reels)
-
- def test_instagram_search_scraper_has_reels_all_method(self):
- """Test Instagram search scraper has reels_all method."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reels_all")
- assert callable(scraper.reels_all)
-
- def test_search_profiles_method_signature(self):
- """Test search profiles method has correct signature."""
- import inspect
-
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.profiles)
-
- # Required: user_name parameter (NOT url)
- assert "user_name" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_search_posts_method_signature(self):
- """Test search posts method has correct signature."""
- import inspect
-
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts)
-
- # Required: url parameter
- assert "url" in sig.parameters
-
- # Optional filters
- assert "num_of_posts" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "post_type" in sig.parameters
- assert "posts_to_not_include" in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults (180s = DEFAULT_TIMEOUT_SHORT)
- assert sig.parameters["timeout"].default == 180
-
- def test_search_reels_method_signature(self):
- """Test search reels method has correct signature."""
- import inspect
-
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reels)
-
- assert "url" in sig.parameters
- assert "num_of_posts" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_search_reels_all_method_signature(self):
- """Test search reels_all method has correct signature."""
- import inspect
-
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.reels_all)
-
- assert "url" in sig.parameters
- assert "num_of_posts" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
-
-class TestInstagramDatasetIDs:
- """Test Instagram has correct dataset IDs."""
-
- def test_scraper_has_all_dataset_ids(self):
- """Test scraper has dataset IDs for all types."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert scraper.DATASET_ID # Default: Profiles
- assert scraper.DATASET_ID_POSTS
- assert scraper.DATASET_ID_COMMENTS
- assert scraper.DATASET_ID_REELS
-
- # All should start with gd_
- assert scraper.DATASET_ID.startswith("gd_")
- assert scraper.DATASET_ID_POSTS.startswith("gd_")
- assert scraper.DATASET_ID_COMMENTS.startswith("gd_")
- assert scraper.DATASET_ID_REELS.startswith("gd_")
-
- def test_search_scraper_has_dataset_ids(self):
- """Test search scraper has dataset IDs."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert scraper.DATASET_ID_PROFILES
- assert scraper.DATASET_ID_POSTS
- assert scraper.DATASET_ID_REELS
-
- assert scraper.DATASET_ID_PROFILES.startswith("gd_")
- assert scraper.DATASET_ID_POSTS.startswith("gd_")
- assert scraper.DATASET_ID_REELS.startswith("gd_")
-
- def test_scraper_has_platform_name(self):
- """Test scraper has correct platform name."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "instagram"
-
- def test_scraper_has_cost_per_record(self):
- """Test scraper has cost per record."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "COST_PER_RECORD")
- assert isinstance(scraper.COST_PER_RECORD, (int, float))
- assert scraper.COST_PER_RECORD > 0
-
-
-class TestInstagramScraperRegistration:
- """Test Instagram scraper is registered correctly."""
-
- def test_instagram_is_registered(self):
- """Test Instagram scraper is in registry."""
- from brightdata.scrapers.registry import is_platform_supported, get_registered_platforms
-
- assert is_platform_supported("instagram")
- assert "instagram" in get_registered_platforms()
-
- def test_can_get_instagram_scraper_from_registry(self):
- """Test can get Instagram scraper from registry."""
- from brightdata.scrapers.registry import get_scraper_for
-
- scraper_class = get_scraper_for("instagram")
- assert scraper_class is not None
- assert scraper_class.__name__ == "InstagramScraper"
-
-
-class TestInstagramClientIntegration:
- """Test Instagram scraper integration with BrightDataClient."""
-
- def test_client_has_instagram_scraper_access(self):
- """Test client provides access to Instagram scraper."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client, "scrape")
- assert hasattr(client.scrape, "instagram")
-
- def test_client_instagram_scraper_has_all_methods(self):
- """Test client.scrape.instagram has all Instagram methods."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.scrape.instagram, "profiles")
- assert hasattr(client.scrape.instagram, "posts")
- assert hasattr(client.scrape.instagram, "comments")
- assert hasattr(client.scrape.instagram, "reels")
-
- def test_instagram_scraper_instance_from_client(self):
- """Test Instagram scraper instance is InstagramScraper."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert isinstance(client.scrape.instagram, InstagramScraper)
-
- def test_client_has_instagram_search_access(self):
- """Test client provides access to Instagram search."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client, "search")
- assert hasattr(client.search, "instagram")
-
- def test_client_instagram_search_has_methods(self):
- """Test client.search.instagram has discovery methods."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.search.instagram, "profiles")
- assert hasattr(client.search.instagram, "posts")
- assert hasattr(client.search.instagram, "reels")
- assert hasattr(client.search.instagram, "reels_all")
-
- def test_instagram_search_instance_from_client(self):
- """Test Instagram search instance is InstagramSearchScraper."""
- client = BrightDataClient(token="test_token_123456789")
-
- assert isinstance(client.search.instagram, InstagramSearchScraper)
-
-
-class TestInstagramScraperConfiguration:
- """Test Instagram scraper configuration."""
-
- def test_scraper_initialization_with_token(self):
- """Test scraper can be initialized with bearer token."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert scraper.bearer_token == "test_token_123456789"
-
- def test_search_scraper_initialization_with_token(self):
- """Test search scraper can be initialized with bearer token."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert scraper.bearer_token == "test_token_123456789"
-
- def test_scraper_has_engine(self):
- """Test scraper has engine instance."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "engine")
- assert scraper.engine is not None
-
- def test_search_scraper_has_engine(self):
- """Test search scraper has engine instance."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "engine")
- assert scraper.engine is not None
-
- def test_scraper_has_api_client(self):
- """Test scraper has API client."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "api_client")
- assert scraper.api_client is not None
-
- def test_scraper_has_workflow_executor(self):
- """Test scraper has workflow executor."""
- scraper = InstagramScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "workflow_executor")
- assert scraper.workflow_executor is not None
-
-
-class TestInstagramScraperExports:
- """Test Instagram scraper is properly exported."""
-
- def test_instagram_scraper_in_module_exports(self):
- """Test InstagramScraper is in scrapers module __all__."""
- from brightdata import scrapers
-
- assert "InstagramScraper" in scrapers.__all__
-
- def test_instagram_search_scraper_in_module_exports(self):
- """Test InstagramSearchScraper is in scrapers module __all__."""
- from brightdata import scrapers
-
- assert "InstagramSearchScraper" in scrapers.__all__
-
- def test_can_import_instagram_scraper_directly(self):
- """Test can import InstagramScraper directly."""
- from brightdata.scrapers import InstagramScraper as IG
-
- assert IG is not None
- assert IG.__name__ == "InstagramScraper"
-
- def test_can_import_instagram_search_scraper_directly(self):
- """Test can import InstagramSearchScraper directly."""
- from brightdata.scrapers import InstagramSearchScraper as IGSearch
-
- assert IGSearch is not None
- assert IGSearch.__name__ == "InstagramSearchScraper"
-
- def test_can_import_from_instagram_submodule(self):
- """Test can import from instagram submodule."""
- from brightdata.scrapers.instagram import InstagramScraper as IG
- from brightdata.scrapers.instagram import InstagramSearchScraper as IGSearch
-
- assert IG is not None
- assert IG.__name__ == "InstagramScraper"
- assert IGSearch is not None
- assert IGSearch.__name__ == "InstagramSearchScraper"
-
-
-class TestInstagramDiscoveryExtraParams:
- """Test Instagram discovery uses extra_params correctly."""
-
- def test_search_scraper_has_execute_discovery_method(self):
- """Test search scraper has internal _execute_discovery method."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "_execute_discovery")
- assert callable(scraper._execute_discovery)
-
- def test_search_scraper_has_context_manager(self):
- """Test search scraper supports async context manager."""
- scraper = InstagramSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "__aenter__")
- assert hasattr(scraper, "__aexit__")
diff --git a/tests/unit/test_linkedin.py b/tests/unit/test_linkedin.py
deleted file mode 100644
index 4e30902..0000000
--- a/tests/unit/test_linkedin.py
+++ /dev/null
@@ -1,535 +0,0 @@
-"""Unit tests for LinkedIn scraper and search services."""
-
-from brightdata import BrightDataClient
-from brightdata.scrapers.linkedin import LinkedInScraper, LinkedInSearchScraper
-
-
-class TestLinkedInScraperURLBased:
- """Test LinkedIn scraper (URL-based extraction)."""
-
- def test_linkedin_scraper_has_posts_method(self):
- """Test LinkedIn scraper has posts method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "posts")
- assert callable(scraper.posts)
-
- def test_linkedin_scraper_has_jobs_method(self):
- """Test LinkedIn scraper has jobs method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "jobs")
- assert callable(scraper.jobs)
-
- def test_linkedin_scraper_has_profiles_method(self):
- """Test LinkedIn scraper has profiles method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "profiles")
- assert callable(scraper.profiles)
-
- def test_linkedin_scraper_has_companies_method(self):
- """Test LinkedIn scraper has companies method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "companies")
- assert callable(scraper.companies)
-
- def test_posts_method_signature(self):
- """Test posts method has correct signature."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts)
-
- # Required: url parameter
- assert "url" in sig.parameters
-
- # Optional: sync and timeout
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
- # Defaults
- assert sig.parameters["timeout"].default == 180
-
- def test_jobs_method_signature(self):
- """Test jobs method has correct signature."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.jobs)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_profiles_method_signature(self):
- """Test profiles method has correct signature."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.profiles)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
- def test_companies_method_signature(self):
- """Test companies method has correct signature."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.companies)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
-
-
-class TestLinkedInSearchScraper:
- """Test LinkedIn search service (discovery/parameter-based)."""
-
- def test_linkedin_search_has_posts_method(self):
- """Test LinkedIn search has posts discovery method (async-first API)."""
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(search, "posts")
- assert callable(search.posts)
-
- def test_linkedin_search_has_profiles_method(self):
- """Test LinkedIn search has profiles discovery method (async-first API)."""
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(search, "profiles")
- assert callable(search.profiles)
-
- def test_linkedin_search_has_jobs_method(self):
- """Test LinkedIn search has jobs discovery method (async-first API)."""
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- assert hasattr(search, "jobs")
- assert callable(search.jobs)
-
- def test_search_posts_signature(self):
- """Test search.posts has correct signature."""
- import inspect
-
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(search.posts)
-
- # Required: url (profile URL)
- assert "url" in sig.parameters
-
- # Optional: start_date, end_date, timeout
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
- assert "timeout" in sig.parameters
-
- def test_search_profiles_signature(self):
- """Test search.profiles has correct signature."""
- import inspect
-
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(search.profiles)
-
- # Required: first_name
- assert "first_name" in sig.parameters
-
- # Optional: last_name, timeout
- assert "last_name" in sig.parameters
- assert "timeout" in sig.parameters
-
- def test_search_jobs_signature(self):
- """Test search.jobs has correct signature."""
- import inspect
-
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(search.jobs)
-
- # All parameters should be present
- params = sig.parameters
- assert "url" in params
- assert "location" in params
- assert "keyword" in params
- assert "country" in params
- assert "timeRange" in params
- assert "jobType" in params
- assert "experienceLevel" in params
- assert "remote" in params
- assert "company" in params
- assert "locationRadius" in params
- assert "timeout" in params
-
-
-class TestLinkedInDualNamespaces:
- """Test LinkedIn has both scrape and search namespaces."""
-
- def test_client_has_scrape_linkedin(self):
- """Test client.scrape.linkedin exists."""
- client = BrightDataClient(token="test_token_123456789")
-
- scraper = client.scrape.linkedin
- assert scraper is not None
- assert isinstance(scraper, LinkedInScraper)
-
- def test_client_has_search_linkedin(self):
- """Test client.search.linkedin exists."""
- client = BrightDataClient(token="test_token_123456789")
-
- search = client.search.linkedin
- assert search is not None
- assert isinstance(search, LinkedInSearchScraper)
-
- def test_scrape_vs_search_distinction(self):
- """Test clear distinction between scrape and search."""
- client = BrightDataClient(token="test_token_123456789")
-
- scraper = client.scrape.linkedin
- search = client.search.linkedin
-
- # Scraper uses 'url' parameter
- import inspect
-
- scraper_sig = inspect.signature(scraper.posts)
- assert "url" in scraper_sig.parameters
- assert "sync" not in scraper_sig.parameters # sync parameter was removed
-
- # Search uses url + date range parameters
- search_sig = inspect.signature(search.posts)
- assert "url" in search_sig.parameters
- assert "start_date" in search_sig.parameters
-
- def test_scrape_linkedin_methods_accept_url_list(self):
- """Test scrape.linkedin methods accept url as str | list."""
- import inspect
-
- client = BrightDataClient(token="test_token_123456789")
- scraper = client.scrape.linkedin
-
- # Check type hints
- sig = inspect.signature(scraper.posts)
- url_param = sig.parameters["url"]
-
- # Should accept Union[str, List[str]]
- annotation_str = str(url_param.annotation)
- assert "str" in annotation_str
- assert "List" in annotation_str or "list" in annotation_str
-
-
-class TestLinkedInDatasetIDs:
- """Test LinkedIn has correct dataset IDs for each type."""
-
- def test_scraper_has_all_dataset_ids(self):
- """Test scraper has dataset IDs for all types."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert scraper.DATASET_ID # Profiles
- assert scraper.DATASET_ID_COMPANIES
- assert scraper.DATASET_ID_JOBS
- assert scraper.DATASET_ID_POSTS
-
- # All should start with gd_
- assert scraper.DATASET_ID.startswith("gd_")
- assert scraper.DATASET_ID_COMPANIES.startswith("gd_")
- assert scraper.DATASET_ID_JOBS.startswith("gd_")
- assert scraper.DATASET_ID_POSTS.startswith("gd_")
-
- def test_search_has_dataset_ids(self):
- """Test search service has dataset IDs."""
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- assert search.DATASET_ID_POSTS
- assert search.DATASET_ID_PROFILES
- assert search.DATASET_ID_JOBS
-
-
-class TestSyncVsAsyncMode:
- """Test sync vs async mode handling."""
-
- def test_default_timeout_is_correct(self):
- """Test default timeout is 180s for async workflow."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts)
-
- assert sig.parameters["timeout"].default == 180
-
- def test_methods_dont_have_sync_parameter(self):
- """Test all scrape methods don't have sync parameter (standard async pattern)."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- for method_name in ["posts", "jobs", "profiles", "companies"]:
- sig = inspect.signature(getattr(scraper, method_name))
- assert "sync" not in sig.parameters
-
-
-class TestAPISpecCompliance:
- """Test compliance with exact API specifications."""
-
- def test_scrape_posts_api_spec(self):
- """Test client.scrape.linkedin.posts matches API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: client.scrape.linkedin.posts(url, timeout=180)
- import inspect
-
- sig = inspect.signature(client.scrape.linkedin.posts)
-
- assert "url" in sig.parameters
- assert "sync" not in sig.parameters
- assert "timeout" in sig.parameters
- assert sig.parameters["timeout"].default == 180
-
- def test_search_posts_api_spec(self):
- """Test client.search.linkedin.posts matches API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: posts(url, start_date, end_date)
- import inspect
-
- sig = inspect.signature(client.search.linkedin.posts)
-
- assert "url" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
-
- def test_search_profiles_api_spec(self):
- """Test client.search.linkedin.profiles matches API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: profiles(first_name, last_name, timeout)
- import inspect
-
- sig = inspect.signature(client.search.linkedin.profiles)
-
- assert "first_name" in sig.parameters
- assert "last_name" in sig.parameters
- assert "timeout" in sig.parameters
-
- def test_search_jobs_api_spec(self):
- """Test client.search.linkedin.jobs matches API spec."""
- client = BrightDataClient(token="test_token_123456789")
-
- # API Spec: jobs(url, location, keyword, country, ...)
- import inspect
-
- sig = inspect.signature(client.search.linkedin.jobs)
-
- params = sig.parameters
- assert "url" in params
- assert "location" in params
- assert "keyword" in params
- assert "country" in params
- assert "timeRange" in params
- assert "jobType" in params
- assert "experienceLevel" in params
- assert "remote" in params
- assert "company" in params
- assert "locationRadius" in params
- assert "timeout" in params
-
-
-class TestLinkedInClientIntegration:
- """Test LinkedIn integrates properly with client."""
-
- def test_linkedin_accessible_via_client_scrape(self):
- """Test LinkedIn scraper accessible via client.scrape.linkedin."""
- client = BrightDataClient(token="test_token_123456789")
-
- linkedin = client.scrape.linkedin
- assert linkedin is not None
- assert isinstance(linkedin, LinkedInScraper)
-
- def test_linkedin_accessible_via_client_search(self):
- """Test LinkedIn search accessible via client.search.linkedin."""
- client = BrightDataClient(token="test_token_123456789")
-
- linkedin_search = client.search.linkedin
- assert linkedin_search is not None
- assert isinstance(linkedin_search, LinkedInSearchScraper)
-
- def test_client_passes_token_to_scraper(self):
- """Test client passes token to LinkedIn scraper."""
- token = "test_token_123456789"
- client = BrightDataClient(token=token)
-
- linkedin = client.scrape.linkedin
- assert linkedin.bearer_token == token
-
- def test_client_passes_token_to_search(self):
- """Test client passes token to LinkedIn search."""
- token = "test_token_123456789"
- client = BrightDataClient(token=token)
-
- search = client.search.linkedin
- assert search.bearer_token == token
-
-
-class TestInterfaceExamples:
- """Test interface examples from specifications."""
-
- def test_scrape_posts_interface(self):
- """Test scrape.linkedin.posts interface."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Interface: posts(url=str|list, timeout=180)
- linkedin = client.scrape.linkedin
-
- # Should be callable
- assert callable(linkedin.posts)
-
- # Accepts url, sync, timeout
- import inspect
-
- sig = inspect.signature(linkedin.posts)
- assert set(["url", "timeout"]).issubset(sig.parameters.keys())
-
- def test_search_posts_interface(self):
- """Test search.linkedin.posts interface."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Interface: posts(url, start_date, end_date)
- linkedin_search = client.search.linkedin
-
- assert callable(linkedin_search.posts)
-
- import inspect
-
- sig = inspect.signature(linkedin_search.posts)
- assert "url" in sig.parameters
- assert "start_date" in sig.parameters
- assert "end_date" in sig.parameters
-
- def test_search_jobs_interface(self):
- """Test search.linkedin.jobs interface."""
- client = BrightDataClient(token="test_token_123456789")
-
- # Interface: jobs(url, location, keyword, ..many filters)
- linkedin_search = client.search.linkedin
-
- assert callable(linkedin_search.jobs)
-
- import inspect
-
- sig = inspect.signature(linkedin_search.jobs)
-
- # All the filters from spec
- expected_params = [
- "url",
- "location",
- "keyword",
- "country",
- "timeRange",
- "jobType",
- "experienceLevel",
- "remote",
- "company",
- "locationRadius",
- "timeout",
- ]
-
- for param in expected_params:
- assert param in sig.parameters
-
-
-class TestParameterArraySupport:
- """Test array parameter support (str | array)."""
-
- def test_url_accepts_string(self):
- """Test url parameter accepts single string."""
- import inspect
-
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(scraper.posts)
-
- # Type annotation should allow str | List[str]
- url_annotation = str(sig.parameters["url"].annotation)
- assert "Union" in url_annotation or "|" in url_annotation
- assert "str" in url_annotation
-
- def test_url_accepts_array_in_search_posts(self):
- """Test url accepts arrays in search posts."""
- import inspect
-
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
- sig = inspect.signature(search.posts)
-
- # url should accept str | list
- annotation = str(sig.parameters["url"].annotation)
- assert "Union" in annotation or "str" in annotation
-
-
-class TestAsyncFirstAPI:
- """Test all methods follow async-first pattern."""
-
- def test_scraper_has_all_methods(self):
- """Test scraper has all methods (async-first API, no _async suffix)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- methods = ["posts", "jobs", "profiles", "companies"]
-
- for method in methods:
- assert hasattr(scraper, method)
- assert callable(getattr(scraper, method))
-
- def test_search_has_all_methods(self):
- """Test search has all methods (async-first API, no _async suffix)."""
- search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- methods = ["posts", "profiles", "jobs"]
-
- for method in methods:
- assert hasattr(search, method)
- assert callable(getattr(search, method))
-
-
-class TestPhilosophicalPrinciples:
- """Test LinkedIn follows philosophical principles."""
-
- def test_clear_scrape_vs_search_distinction(self):
- """Test clear distinction between scrape (URL) and search (params)."""
- client = BrightDataClient(token="test_token_123456789")
-
- scraper = client.scrape.linkedin
- search = client.search.linkedin
-
- # Scraper is for URLs
- import inspect
-
- scraper_posts_sig = inspect.signature(scraper.posts)
- assert "url" in scraper_posts_sig.parameters
-
- # Search is for discovery parameters (url + date range)
- search_posts_sig = inspect.signature(search.posts)
- assert "url" in search_posts_sig.parameters
- assert "start_date" in search_posts_sig.parameters
-
- def test_consistent_timeout_defaults(self):
- """Test consistent timeout defaults across methods."""
- client = BrightDataClient(token="test_token_123456789")
-
- scraper = client.scrape.linkedin
-
- import inspect
-
- # All scrape methods should default to 65s
- for method_name in ["posts", "jobs", "profiles", "companies"]:
- sig = inspect.signature(getattr(scraper, method_name))
- assert sig.parameters["timeout"].default == 180
-
- def test_uses_standard_async_workflow(self):
- """Test methods use standard async workflow (no sync parameter)."""
- client = BrightDataClient(token="test_token_123456789")
-
- scraper = client.scrape.linkedin
-
- import inspect
-
- sig = inspect.signature(scraper.posts)
-
- # Should not have sync parameter
- assert "sync" not in sig.parameters
diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py
index 1ac5fa7..3d7ee11 100644
--- a/tests/unit/test_models.py
+++ b/tests/unit/test_models.py
@@ -1,6 +1,9 @@
-"""Unit tests for result models."""
+"""Tests for result models — Creation, timing, serialization, and method tracking."""
+import json
from datetime import datetime, timezone
+
+
from brightdata.models import (
BaseResult,
ScrapeResult,
@@ -9,61 +12,46 @@
)
-class TestBaseResult:
- """Tests for BaseResult class."""
+# ---------------------------------------------------------------------------
+# BaseResult
+# ---------------------------------------------------------------------------
- def test_creation(self):
- """Test basic creation of BaseResult."""
+
+class TestBaseResult:
+ def test_creation_defaults(self):
result = BaseResult(success=True)
assert result.success is True
assert result.cost is None
assert result.error is None
- def test_elapsed_ms(self):
- """Test elapsed time calculation."""
+ def test_elapsed_ms_zero_delta(self):
now = datetime.now(timezone.utc)
- result = BaseResult(
- success=True,
- trigger_sent_at=now,
- data_fetched_at=now,
- )
+ result = BaseResult(success=True, trigger_sent_at=now, data_fetched_at=now)
elapsed = result.elapsed_ms()
assert elapsed is not None
assert elapsed >= 0
- def test_elapsed_ms_with_delta(self):
- """Test elapsed time with actual time difference."""
+ def test_elapsed_ms_one_second(self):
start = datetime(2025, 1, 1, 12, 0, 0)
end = datetime(2025, 1, 1, 12, 0, 1)
- result = BaseResult(
- success=True,
- trigger_sent_at=start,
- data_fetched_at=end,
- )
+ result = BaseResult(success=True, trigger_sent_at=start, data_fetched_at=end)
assert result.elapsed_ms() == 1000.0
- def test_get_timing_breakdown(self):
- """Test timing breakdown generation."""
+ def test_timing_breakdown_keys(self):
now = datetime.now(timezone.utc)
- result = BaseResult(
- success=True,
- trigger_sent_at=now,
- data_fetched_at=now,
- )
+ result = BaseResult(success=True, trigger_sent_at=now, data_fetched_at=now)
breakdown = result.get_timing_breakdown()
assert "total_elapsed_ms" in breakdown
assert "trigger_sent_at" in breakdown
assert "data_fetched_at" in breakdown
def test_to_dict(self):
- """Test conversion to dictionary."""
result = BaseResult(success=True, cost=0.001)
data = result.to_dict()
assert data["success"] is True
assert data["cost"] == 0.001
def test_to_json(self):
- """Test JSON serialization."""
result = BaseResult(success=True, cost=0.001)
json_str = result.to_json()
assert isinstance(json_str, str)
@@ -71,7 +59,6 @@ def test_to_json(self):
assert "0.001" in json_str
def test_save_to_file(self, tmp_path):
- """Test saving to file."""
result = BaseResult(success=True, cost=0.001)
filepath = tmp_path / "result.json"
result.save_to_file(filepath)
@@ -82,22 +69,19 @@ def test_save_to_file(self, tmp_path):
assert "0.001" in content
-class TestScrapeResult:
- """Tests for ScrapeResult class."""
+# ---------------------------------------------------------------------------
+# ScrapeResult
+# ---------------------------------------------------------------------------
+
+class TestScrapeResult:
def test_creation(self):
- """Test basic creation of ScrapeResult."""
- result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- )
+ result = ScrapeResult(success=True, url="https://example.com", status="ready")
assert result.success is True
assert result.url == "https://example.com"
assert result.status == "ready"
def test_with_platform(self):
- """Test ScrapeResult with platform."""
result = ScrapeResult(
success=True,
url="https://www.linkedin.com/in/test",
@@ -107,7 +91,6 @@ def test_with_platform(self):
assert result.platform == "linkedin"
def test_timing_breakdown_with_polling(self):
- """Test timing breakdown includes polling information."""
start = datetime(2025, 1, 1, 12, 0, 0)
snapshot_received = datetime(2025, 1, 1, 12, 0, 1)
end = datetime(2025, 1, 1, 12, 0, 5)
@@ -128,22 +111,19 @@ def test_timing_breakdown_with_polling(self):
assert breakdown["poll_count"] == 2
-class TestSearchResult:
- """Tests for SearchResult class."""
+# ---------------------------------------------------------------------------
+# SearchResult
+# ---------------------------------------------------------------------------
+
+class TestSearchResult:
def test_creation(self):
- """Test basic creation of SearchResult."""
- query = {"q": "python", "engine": "google"}
- result = SearchResult(
- success=True,
- query=query,
- )
+ result = SearchResult(success=True, query={"q": "python", "engine": "google"})
assert result.success is True
- assert result.query == query
+ assert result.query == {"q": "python", "engine": "google"}
assert result.total_found is None
def test_with_total_found(self):
- """Test SearchResult with total results."""
result = SearchResult(
success=True,
query={"q": "python"},
@@ -154,36 +134,28 @@ def test_with_total_found(self):
assert result.search_engine == "google"
-class TestCrawlResult:
- """Tests for CrawlResult class."""
+# ---------------------------------------------------------------------------
+# CrawlResult
+# ---------------------------------------------------------------------------
+
+class TestCrawlResult:
def test_creation(self):
- """Test basic creation of CrawlResult."""
- result = CrawlResult(
- success=True,
- domain="example.com",
- )
+ result = CrawlResult(success=True, domain="example.com")
assert result.success is True
assert result.domain == "example.com"
assert result.pages == []
def test_with_pages(self):
- """Test CrawlResult with crawled pages."""
pages = [
{"url": "https://example.com/page1", "data": {}},
{"url": "https://example.com/page2", "data": {}},
]
- result = CrawlResult(
- success=True,
- domain="example.com",
- pages=pages,
- total_pages=2,
- )
+ result = CrawlResult(success=True, domain="example.com", pages=pages, total_pages=2)
assert len(result.pages) == 2
assert result.total_pages == 2
def test_timing_breakdown_with_crawl_duration(self):
- """Test timing breakdown includes crawl duration."""
crawl_start = datetime(2025, 1, 1, 12, 0, 0)
crawl_end = datetime(2025, 1, 1, 12, 5, 0)
@@ -199,170 +171,77 @@ def test_timing_breakdown_with_crawl_duration(self):
assert breakdown["crawl_duration_ms"] == 300000.0
-class TestInterfaceRequirements:
- """Test all interface requirements are met."""
-
- def test_common_fields(self):
- """Test common fields across all results."""
- result = BaseResult(success=True, cost=0.001, error=None)
- assert hasattr(result, "success")
- assert hasattr(result, "cost")
- assert hasattr(result, "error")
- assert hasattr(result, "trigger_sent_at")
- assert hasattr(result, "data_fetched_at")
-
- def test_common_methods(self):
- """Test common methods across all results."""
- result = BaseResult(success=True)
- assert hasattr(result, "elapsed_ms")
- assert hasattr(result, "to_json")
- assert hasattr(result, "save_to_file")
- assert hasattr(result, "get_timing_breakdown")
-
- def test_scrape_specific_fields(self):
- """Test ScrapeResult specific fields."""
- scrape = ScrapeResult(success=True, url="https://example.com", status="ready")
- assert hasattr(scrape, "url")
- assert hasattr(scrape, "platform")
- assert hasattr(scrape, "method")
-
- def test_search_specific_fields(self):
- """Test SearchResult specific fields."""
- search = SearchResult(success=True, query={"q": "test"})
- assert hasattr(search, "query")
- assert hasattr(search, "total_found")
-
- def test_crawl_specific_fields(self):
- """Test CrawlResult specific fields."""
- crawl = CrawlResult(success=True, domain="example.com")
- assert hasattr(crawl, "domain")
- assert hasattr(crawl, "pages")
+# ---------------------------------------------------------------------------
+# Method field tracking
+# ---------------------------------------------------------------------------
class TestMethodFieldTracking:
- """Tests for method field tracking in results."""
-
- def test_scrape_result_accepts_method_parameter(self):
- """Test ScrapeResult accepts method parameter."""
+ def test_accepts_method_parameter(self):
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="web_scraper",
+ success=True, url="https://example.com", status="ready", method="web_scraper"
)
assert result.method == "web_scraper"
- def test_scrape_result_method_can_be_web_unlocker(self):
- """Test ScrapeResult method can be 'web_unlocker'."""
+ def test_method_web_unlocker(self):
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="web_unlocker",
+ success=True, url="https://example.com", status="ready", method="web_unlocker"
)
assert result.method == "web_unlocker"
- def test_scrape_result_method_can_be_browser_api(self):
- """Test ScrapeResult method can be 'browser_api'."""
+ def test_method_browser_api(self):
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="browser_api",
+ success=True, url="https://example.com", status="ready", method="browser_api"
)
assert result.method == "browser_api"
- def test_scrape_result_method_defaults_to_none(self):
- """Test ScrapeResult method defaults to None."""
- result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- )
+ def test_method_defaults_to_none(self):
+ result = ScrapeResult(success=True, url="https://example.com", status="ready")
assert result.method is None
- def test_method_included_in_to_dict(self):
- """Test method field is included in to_dict output."""
+ def test_method_in_to_dict(self):
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="web_scraper",
+ success=True, url="https://example.com", status="ready", method="web_scraper"
)
data = result.to_dict()
- assert "method" in data
assert data["method"] == "web_scraper"
- def test_method_included_in_json(self):
- """Test method field is included in JSON output."""
+ def test_method_in_json(self):
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="web_unlocker",
+ success=True, url="https://example.com", status="ready", method="web_unlocker"
)
json_str = result.to_json()
- assert "method" in json_str
assert "web_unlocker" in json_str
def test_method_persists_through_serialization(self):
- """Test method field persists through serialization."""
- import json
-
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method="browser_api",
+ success=True, url="https://example.com", status="ready", method="browser_api"
)
-
- # Serialize to dict and back
data = result.to_dict()
assert data["method"] == "browser_api"
- # Serialize to JSON and parse
- json_str = result.to_json()
- parsed = json.loads(json_str)
+ parsed = json.loads(result.to_json())
assert parsed["method"] == "browser_api"
-
-class TestMethodFieldIntegration:
- """Test method field integration with scrapers."""
-
- def test_method_field_tracks_scraping_approach(self):
- """Test method field effectively tracks scraping approach."""
- # Test all three methods
- methods = ["web_scraper", "web_unlocker", "browser_api"]
-
- for method in methods:
+ def test_all_methods_valid(self):
+ for method in ["web_scraper", "web_unlocker", "browser_api"]:
result = ScrapeResult(
- success=True,
- url="https://example.com",
- status="ready",
- method=method,
+ success=True, url="https://example.com", status="ready", method=method
)
assert result.method == method
- assert result.method in ["web_scraper", "web_unlocker", "browser_api"]
- def test_method_field_helps_identify_data_source(self):
- """Test method field helps identify data source."""
- # Different methods might have different characteristics
- web_scraper = ScrapeResult(
+ def test_method_distinguishes_data_source(self):
+ ws = ScrapeResult(
success=True,
url="https://example.com",
status="ready",
method="web_scraper",
platform="linkedin",
)
-
- web_unlocker = ScrapeResult(
+ wu = ScrapeResult(
success=True,
url="https://example.com",
status="ready",
method="web_unlocker",
)
-
- # Both valid, but method provides context
- assert web_scraper.method == "web_scraper"
- assert web_unlocker.method == "web_unlocker"
- assert web_scraper.method != web_unlocker.method
+ assert ws.method != wu.method
diff --git a/tests/unit/test_payloads.py b/tests/unit/test_payloads.py
index 3282657..6d764ed 100644
--- a/tests/unit/test_payloads.py
+++ b/tests/unit/test_payloads.py
@@ -1,13 +1,4 @@
-"""
-Tests for dataclass-based payloads.
-
-Tests validate:
-- Runtime validation
-- Default values
-- Helper methods and properties
-- Error handling
-- Conversion to dict
-"""
+"""Tests for payload dataclasses — Validation, defaults, and serialization."""
import pytest
from brightdata.payloads import (
@@ -32,15 +23,16 @@
)
-class TestAmazonPayloads:
- """Test Amazon payload dataclasses."""
+# ---------------------------------------------------------------------------
+# Amazon
+# ---------------------------------------------------------------------------
+
- def test_amazon_product_payload_valid(self):
- """Test valid Amazon product payload."""
+class TestAmazonPayloads:
+ def test_product_payload_valid(self):
payload = AmazonProductPayload(
url="https://amazon.com/dp/B0CRMZHDG8", reviews_count=50, images_count=10
)
-
assert payload.url == "https://amazon.com/dp/B0CRMZHDG8"
assert payload.reviews_count == 50
assert payload.images_count == 10
@@ -49,296 +41,248 @@ def test_amazon_product_payload_valid(self):
assert payload.domain == "amazon.com"
assert payload.is_secure is True
- def test_amazon_product_payload_defaults(self):
- """Test Amazon product payload with defaults."""
+ def test_product_payload_defaults(self):
payload = AmazonProductPayload(url="https://amazon.com/dp/B123456789")
-
assert payload.reviews_count is None
assert payload.images_count is None
- def test_amazon_product_payload_invalid_url(self):
- """Test Amazon product payload with invalid URL."""
+ def test_product_payload_rejects_non_amazon_url(self):
with pytest.raises(ValueError, match="url must be an Amazon URL"):
AmazonProductPayload(url="https://ebay.com/item/123")
- def test_amazon_product_payload_negative_count(self):
- """Test Amazon product payload with negative count."""
+ def test_product_payload_rejects_negative_count(self):
with pytest.raises(ValueError, match="reviews_count must be non-negative"):
AmazonProductPayload(url="https://amazon.com/dp/B123", reviews_count=-1)
- def test_amazon_product_payload_to_dict(self):
- """Test converting Amazon product payload to dict."""
+ def test_product_payload_to_dict_excludes_none(self):
payload = AmazonProductPayload(url="https://amazon.com/dp/B123", reviews_count=50)
-
result = payload.to_dict()
assert result == {"url": "https://amazon.com/dp/B123", "reviews_count": 50}
- # images_count (None) should not be in dict
assert "images_count" not in result
- def test_amazon_review_payload_valid(self):
- """Test valid Amazon review payload."""
+ def test_review_payload_valid(self):
payload = AmazonReviewPayload(
url="https://amazon.com/dp/B123", pastDays=30, keyWord="quality", numOfReviews=100
)
-
assert payload.pastDays == 30
assert payload.keyWord == "quality"
assert payload.numOfReviews == 100
-class TestLinkedInPayloads:
- """Test LinkedIn payload dataclasses."""
+# ---------------------------------------------------------------------------
+# LinkedIn
+# ---------------------------------------------------------------------------
- def test_linkedin_profile_payload_valid(self):
- """Test valid LinkedIn profile payload."""
- payload = LinkedInProfilePayload(url="https://linkedin.com/in/johndoe")
+class TestLinkedInPayloads:
+ def test_profile_payload_valid(self):
+ payload = LinkedInProfilePayload(url="https://linkedin.com/in/johndoe")
assert payload.url == "https://linkedin.com/in/johndoe"
assert "linkedin.com" in payload.domain
- def test_linkedin_profile_payload_invalid_url(self):
- """Test LinkedIn profile payload with invalid URL."""
+ def test_profile_payload_rejects_non_linkedin_url(self):
with pytest.raises(ValueError, match="url must be a LinkedIn URL"):
LinkedInProfilePayload(url="https://facebook.com/johndoe")
- def test_linkedin_profile_search_payload_valid(self):
- """Test valid LinkedIn profile search payload."""
+ def test_profile_search_payload_valid(self):
payload = LinkedInProfileSearchPayload(firstName="John", lastName="Doe", company="Google")
-
assert payload.firstName == "John"
assert payload.lastName == "Doe"
assert payload.company == "Google"
- def test_linkedin_profile_search_payload_empty_firstname(self):
- """Test LinkedIn profile search with empty firstName."""
+ def test_profile_search_rejects_empty_firstname(self):
with pytest.raises(ValueError, match="firstName is required"):
LinkedInProfileSearchPayload(firstName="")
- def test_linkedin_job_search_payload_valid(self):
- """Test valid LinkedIn job search payload."""
+ def test_job_search_payload_valid(self):
payload = LinkedInJobSearchPayload(
keyword="python developer", location="New York", remote=True, experienceLevel="mid"
)
-
assert payload.keyword == "python developer"
assert payload.location == "New York"
assert payload.remote is True
assert payload.is_remote_search is True
- def test_linkedin_job_search_payload_no_criteria(self):
- """Test LinkedIn job search with no search criteria."""
+ def test_job_search_rejects_no_criteria(self):
with pytest.raises(ValueError, match="At least one search parameter required"):
LinkedInJobSearchPayload()
- def test_linkedin_job_search_payload_invalid_country(self):
- """Test LinkedIn job search with invalid country code."""
+ def test_job_search_rejects_invalid_country_code(self):
with pytest.raises(ValueError, match="country must be 2-letter code"):
- LinkedInJobSearchPayload(keyword="python", country="USA") # Should be "US"
+ LinkedInJobSearchPayload(keyword="python", country="USA")
- def test_linkedin_post_search_payload_valid(self):
- """Test valid LinkedIn post search payload."""
+ def test_post_search_payload_valid(self):
payload = LinkedInPostSearchPayload(
url="https://linkedin.com/in/johndoe", start_date="2025-01-01", end_date="2025-12-31"
)
-
assert payload.start_date == "2025-01-01"
assert payload.end_date == "2025-12-31"
- def test_linkedin_post_search_payload_invalid_date(self):
- """Test LinkedIn post search with invalid date format."""
+ def test_post_search_rejects_invalid_date_format(self):
with pytest.raises(ValueError, match="start_date must be in yyyy-mm-dd format"):
LinkedInPostSearchPayload(
- url="https://linkedin.com/in/johndoe", start_date="01-01-2025" # Wrong format
+ url="https://linkedin.com/in/johndoe", start_date="01-01-2025"
)
-class TestChatGPTPayloads:
- """Test ChatGPT payload dataclasses."""
+# ---------------------------------------------------------------------------
+# ChatGPT
+# ---------------------------------------------------------------------------
+
- def test_chatgpt_prompt_payload_valid(self):
- """Test valid ChatGPT prompt payload."""
+class TestChatGPTPayloads:
+ def test_prompt_payload_valid(self):
payload = ChatGPTPromptPayload(
prompt="Explain Python async programming", country="US", web_search=True
)
-
assert payload.prompt == "Explain Python async programming"
assert payload.country == "US"
assert payload.web_search is True
assert payload.uses_web_search is True
- def test_chatgpt_prompt_payload_defaults(self):
- """Test ChatGPT prompt payload defaults."""
+ def test_prompt_payload_defaults(self):
payload = ChatGPTPromptPayload(prompt="Test prompt")
-
assert payload.country == "US"
assert payload.web_search is False
assert payload.additional_prompt is None
- def test_chatgpt_prompt_payload_empty_prompt(self):
- """Test ChatGPT payload with empty prompt."""
+ def test_prompt_payload_rejects_empty_prompt(self):
with pytest.raises(ValueError, match="prompt is required"):
ChatGPTPromptPayload(prompt="")
- def test_chatgpt_prompt_payload_invalid_country(self):
- """Test ChatGPT payload with invalid country code."""
+ def test_prompt_payload_rejects_invalid_country(self):
with pytest.raises(ValueError, match="country must be 2-letter code"):
- ChatGPTPromptPayload(prompt="Test", country="USA") # Should be "US"
+ ChatGPTPromptPayload(prompt="Test", country="USA")
- def test_chatgpt_prompt_payload_too_long(self):
- """Test ChatGPT payload with prompt too long."""
+ def test_prompt_payload_rejects_too_long(self):
with pytest.raises(ValueError, match="prompt too long"):
ChatGPTPromptPayload(prompt="x" * 10001)
-class TestFacebookPayloads:
- """Test Facebook payload dataclasses."""
+# ---------------------------------------------------------------------------
+# Facebook
+# ---------------------------------------------------------------------------
+
- def test_facebook_posts_profile_payload_valid(self):
- """Test valid Facebook posts profile payload."""
+class TestFacebookPayloads:
+ def test_posts_profile_payload_valid(self):
payload = FacebookPostsProfilePayload(
url="https://facebook.com/profile",
num_of_posts=10,
start_date="01-01-2025",
end_date="12-31-2025",
)
-
assert payload.url == "https://facebook.com/profile"
assert payload.num_of_posts == 10
assert payload.start_date == "01-01-2025"
- def test_facebook_posts_profile_payload_invalid_url(self):
- """Test Facebook payload with invalid URL."""
+ def test_posts_profile_rejects_non_facebook_url(self):
with pytest.raises(ValueError, match="url must be a Facebook URL"):
FacebookPostsProfilePayload(url="https://twitter.com/user")
- def test_facebook_posts_group_payload_valid(self):
- """Test valid Facebook posts group payload."""
+ def test_posts_group_payload_valid(self):
payload = FacebookPostsGroupPayload(
url="https://facebook.com/groups/example", num_of_posts=20
)
-
assert payload.url == "https://facebook.com/groups/example"
assert payload.num_of_posts == 20
- def test_facebook_posts_group_payload_not_group(self):
- """Test Facebook group payload without /groups/ in URL."""
+ def test_posts_group_rejects_non_group_url(self):
with pytest.raises(ValueError, match="url must be a Facebook group URL"):
FacebookPostsGroupPayload(url="https://facebook.com/profile")
- def test_facebook_comments_payload_valid(self):
- """Test valid Facebook comments payload."""
+ def test_comments_payload_valid(self):
payload = FacebookCommentsPayload(
url="https://facebook.com/post/123456", num_of_comments=100
)
-
assert payload.num_of_comments == 100
-class TestInstagramPayloads:
- """Test Instagram payload dataclasses."""
+# ---------------------------------------------------------------------------
+# Instagram
+# ---------------------------------------------------------------------------
- def test_instagram_profile_payload_valid(self):
- """Test valid Instagram profile payload."""
- payload = InstagramProfilePayload(url="https://instagram.com/username")
+class TestInstagramPayloads:
+ def test_profile_payload_valid(self):
+ payload = InstagramProfilePayload(url="https://instagram.com/username")
assert payload.url == "https://instagram.com/username"
assert "instagram.com" in payload.domain
- def test_instagram_post_payload_valid(self):
- """Test valid Instagram post payload."""
+ def test_post_payload_valid(self):
payload = InstagramPostPayload(url="https://instagram.com/p/ABC123")
-
assert payload.url == "https://instagram.com/p/ABC123"
assert payload.is_post is True
- def test_instagram_reel_payload_valid(self):
- """Test valid Instagram reel payload."""
+ def test_reel_payload_valid(self):
payload = InstagramReelPayload(url="https://instagram.com/reel/ABC123")
-
assert payload.url == "https://instagram.com/reel/ABC123"
assert payload.is_reel is True
- def test_instagram_posts_discover_payload_valid(self):
- """Test valid Instagram posts discover payload."""
+ def test_posts_discover_payload_valid(self):
payload = InstagramPostsDiscoverPayload(
url="https://instagram.com/username", num_of_posts=10, post_type="reel"
)
-
assert payload.num_of_posts == 10
assert payload.post_type == "reel"
- def test_instagram_posts_discover_payload_invalid_count(self):
- """Test Instagram discover payload with invalid count."""
+ def test_posts_discover_rejects_zero_count(self):
with pytest.raises(ValueError, match="num_of_posts must be positive"):
InstagramPostsDiscoverPayload(url="https://instagram.com/username", num_of_posts=0)
-class TestBasePayload:
- """Test base payload functionality."""
+# ---------------------------------------------------------------------------
+# Base payload behavior
+# ---------------------------------------------------------------------------
- def test_url_payload_invalid_type(self):
- """Test URL payload with invalid type."""
+
+class TestBasePayloadBehavior:
+ def test_rejects_non_string_url(self):
with pytest.raises(TypeError, match="url must be string"):
AmazonProductPayload(url=123) # type: ignore
- def test_url_payload_empty(self):
- """Test URL payload with empty string."""
+ def test_rejects_empty_url(self):
with pytest.raises(ValueError, match="url cannot be empty"):
AmazonProductPayload(url="")
- def test_url_payload_no_protocol(self):
- """Test URL payload without protocol."""
+ def test_rejects_url_without_protocol(self):
with pytest.raises(ValueError, match="url must be valid HTTP/HTTPS URL"):
AmazonProductPayload(url="amazon.com/dp/B123")
- def test_url_payload_properties(self):
- """Test URL payload helper properties."""
+ def test_url_helper_properties(self):
payload = AmazonProductPayload(url="https://amazon.com/dp/B123")
-
assert payload.domain == "amazon.com"
assert payload.is_secure is True
- # Test non-HTTPS
payload_http = FacebookPostPayload(url="http://facebook.com/post/123")
assert payload_http.is_secure is False
- def test_to_dict_excludes_none(self):
- """Test to_dict() excludes None values."""
- payload = AmazonProductPayload(
- url="https://amazon.com/dp/B123",
- reviews_count=50,
- # images_count not provided (None)
- )
-
+ def test_to_dict_excludes_none_values(self):
+ payload = AmazonProductPayload(url="https://amazon.com/dp/B123", reviews_count=50)
result = payload.to_dict()
assert "images_count" not in result
assert "reviews_count" in result
-class TestPayloadIntegration:
- """Integration tests for payload usage."""
+# ---------------------------------------------------------------------------
+# Integration
+# ---------------------------------------------------------------------------
+
- def test_payload_lifecycle(self):
- """Test complete payload lifecycle."""
- # Create payload with validation
+class TestPayloadIntegration:
+ def test_full_lifecycle(self):
payload = LinkedInJobSearchPayload(
keyword="python developer", location="New York", remote=True
)
-
- # Check properties work
assert payload.is_remote_search is True
- # Convert to dict for API call
api_dict = payload.to_dict()
assert api_dict["keyword"] == "python developer"
assert api_dict["remote"] is True
-
- # Verify None values excluded
assert "url" not in api_dict
assert "company" not in api_dict
- def test_multiple_payloads_consistency(self):
- """Test consistency across different payload types."""
+ def test_consistent_interface_across_types(self):
payloads = [
AmazonProductPayload(url="https://amazon.com/dp/B123"),
LinkedInProfilePayload(url="https://linkedin.com/in/johndoe"),
@@ -346,7 +290,6 @@ def test_multiple_payloads_consistency(self):
InstagramPostPayload(url="https://instagram.com/p/ABC123"),
]
- # All should have consistent interface
for payload in payloads:
assert hasattr(payload, "url")
assert hasattr(payload, "domain")
diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py
index cf6590a..2ee48ae 100644
--- a/tests/unit/test_retry.py
+++ b/tests/unit/test_retry.py
@@ -1 +1,180 @@
-"""Unit tests for retry logic."""
+"""Tests for utils/retry.py — Exponential backoff logic."""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from brightdata.utils.retry import retry_with_backoff
+from brightdata.exceptions import APIError, NetworkError, AuthenticationError, ValidationError
+
+
+# ---------------------------------------------------------------------------
+# Happy path
+# ---------------------------------------------------------------------------
+
+
+class TestRetrySuccess:
+ @pytest.mark.asyncio
+ async def test_returns_on_first_success(self):
+ func = AsyncMock(return_value="ok")
+ result = await retry_with_backoff(func, max_retries=3)
+ assert result == "ok"
+ assert func.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_retries_then_succeeds(self):
+ func = AsyncMock(side_effect=[NetworkError("fail"), NetworkError("fail"), "ok"])
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ result = await retry_with_backoff(func, max_retries=3, initial_delay=0.01)
+
+ assert result == "ok"
+ assert func.call_count == 3
+
+
+# ---------------------------------------------------------------------------
+# Retryable vs non-retryable exceptions
+# ---------------------------------------------------------------------------
+
+
+class TestRetryableExceptions:
+ @pytest.mark.asyncio
+ async def test_retries_on_network_error(self):
+ func = AsyncMock(side_effect=[NetworkError("net"), "ok"])
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ result = await retry_with_backoff(func, max_retries=3, initial_delay=0.01)
+
+ assert result == "ok"
+
+ @pytest.mark.asyncio
+ async def test_retries_on_timeout_error(self):
+ func = AsyncMock(side_effect=[TimeoutError("timeout"), "ok"])
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ result = await retry_with_backoff(func, max_retries=3, initial_delay=0.01)
+
+ assert result == "ok"
+
+ @pytest.mark.asyncio
+ async def test_retries_on_api_error(self):
+ func = AsyncMock(side_effect=[APIError("500", status_code=500), "ok"])
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ result = await retry_with_backoff(func, max_retries=3, initial_delay=0.01)
+
+ assert result == "ok"
+
+ @pytest.mark.asyncio
+ async def test_does_not_retry_authentication_error(self):
+ func = AsyncMock(side_effect=AuthenticationError("bad token"))
+
+ with pytest.raises(AuthenticationError, match="bad token"):
+ await retry_with_backoff(func, max_retries=3)
+
+ assert func.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_does_not_retry_validation_error(self):
+ func = AsyncMock(side_effect=ValidationError("bad input"))
+
+ with pytest.raises(ValidationError, match="bad input"):
+ await retry_with_backoff(func, max_retries=3)
+
+ assert func.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_does_not_retry_value_error(self):
+ func = AsyncMock(side_effect=ValueError("wrong"))
+
+ with pytest.raises(ValueError):
+ await retry_with_backoff(func, max_retries=3)
+
+ assert func.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_custom_retryable_exceptions(self):
+ func = AsyncMock(side_effect=[ValueError("retry me"), "ok"])
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ result = await retry_with_backoff(
+ func,
+ max_retries=3,
+ retryable_exceptions=[ValueError],
+ initial_delay=0.01,
+ )
+
+ assert result == "ok"
+
+
+# ---------------------------------------------------------------------------
+# Exhausted retries
+# ---------------------------------------------------------------------------
+
+
+class TestRetryExhausted:
+ @pytest.mark.asyncio
+ async def test_raises_last_exception_after_max_retries(self):
+ func = AsyncMock(side_effect=NetworkError("persistent failure"))
+
+ with patch("brightdata.utils.retry.asyncio.sleep", new_callable=AsyncMock):
+ with pytest.raises(NetworkError, match="persistent failure"):
+ await retry_with_backoff(func, max_retries=2, initial_delay=0.01)
+
+ assert func.call_count == 3 # initial + 2 retries
+
+ @pytest.mark.asyncio
+ async def test_zero_retries_calls_once(self):
+ func = AsyncMock(side_effect=NetworkError("fail"))
+
+ with pytest.raises(NetworkError):
+ await retry_with_backoff(func, max_retries=0)
+
+ assert func.call_count == 1
+
+
+# ---------------------------------------------------------------------------
+# Backoff timing
+# ---------------------------------------------------------------------------
+
+
+class TestBackoffTiming:
+ @pytest.mark.asyncio
+ async def test_exponential_backoff_delays(self):
+ func = AsyncMock(
+ side_effect=[NetworkError("1"), NetworkError("2"), NetworkError("3"), "ok"]
+ )
+ sleep_calls = []
+
+ async def mock_sleep(duration):
+ sleep_calls.append(duration)
+
+ with patch("brightdata.utils.retry.asyncio.sleep", side_effect=mock_sleep):
+ result = await retry_with_backoff(
+ func, max_retries=3, initial_delay=1.0, backoff_factor=2.0
+ )
+
+ assert result == "ok"
+ assert sleep_calls == [1.0, 2.0, 4.0]
+
+ @pytest.mark.asyncio
+ async def test_max_delay_cap(self):
+ func = AsyncMock(
+ side_effect=[NetworkError("1"), NetworkError("2"), NetworkError("3"), "ok"]
+ )
+ sleep_calls = []
+
+ async def mock_sleep(duration):
+ sleep_calls.append(duration)
+
+ with patch("brightdata.utils.retry.asyncio.sleep", side_effect=mock_sleep):
+ result = await retry_with_backoff(
+ func,
+ max_retries=3,
+ initial_delay=10.0,
+ backoff_factor=10.0,
+ max_delay=50.0,
+ )
+
+ # Delays: min(10, 50)=10, min(100, 50)=50, min(1000, 50)=50
+ assert sleep_calls == [10.0, 50.0, 50.0]
diff --git a/tests/unit/test_scrapers.py b/tests/unit/test_scrapers.py
deleted file mode 100644
index c999ff7..0000000
--- a/tests/unit/test_scrapers.py
+++ /dev/null
@@ -1,476 +0,0 @@
-"""Unit tests for base scraper and platform scrapers."""
-
-import pytest
-from unittest.mock import patch
-from brightdata.scrapers import (
- BaseWebScraper,
- AmazonScraper,
- LinkedInScraper,
- ChatGPTScraper,
- register,
- get_scraper_for,
- get_registered_platforms,
- is_platform_supported,
-)
-from brightdata.exceptions import ValidationError
-
-
-class TestBaseWebScraper:
- """Test BaseWebScraper abstract base class."""
-
- def test_base_scraper_requires_dataset_id(self):
- """Test base scraper requires DATASET_ID to be defined."""
-
- class TestScraper(BaseWebScraper):
- # Missing DATASET_ID
- pass
-
- with pytest.raises(NotImplementedError) as exc_info:
- TestScraper(bearer_token="test_token_123456789")
-
- assert "DATASET_ID" in str(exc_info.value)
-
- def test_base_scraper_requires_token(self):
- """Test base scraper requires bearer token."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_dataset_123"
-
- with patch.dict("os.environ", {}, clear=True):
- with pytest.raises(ValidationError) as exc_info:
- TestScraper()
-
- assert "token" in str(exc_info.value).lower()
-
- def test_base_scraper_accepts_token_from_env(self):
- """Test base scraper loads token from environment."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_dataset_123"
- PLATFORM_NAME = "test"
-
- with patch.dict("os.environ", {"BRIGHTDATA_API_TOKEN": "env_token_123456789"}):
- scraper = TestScraper()
- assert scraper.bearer_token == "env_token_123456789"
-
- def test_base_scraper_has_required_attributes(self):
- """Test base scraper has all required class attributes."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_123"
- PLATFORM_NAME = "test"
-
- scraper = TestScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "DATASET_ID")
- assert hasattr(scraper, "PLATFORM_NAME")
- assert hasattr(scraper, "MIN_POLL_TIMEOUT")
- assert hasattr(scraper, "COST_PER_RECORD")
- assert hasattr(scraper, "engine")
-
- def test_base_scraper_has_scrape_methods(self):
- """Test base scraper has scrape methods (async-first API)."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_123"
-
- scraper = TestScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "scrape")
- assert callable(scraper.scrape)
-
- def test_base_scraper_has_normalize_result_method(self):
- """Test base scraper has normalize_result method."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_123"
-
- scraper = TestScraper(bearer_token="test_token_123456789")
-
- # Should return data as-is by default
- test_data = {"key": "value"}
- normalized = scraper.normalize_result(test_data)
- assert normalized == test_data
-
- def test_base_scraper_repr(self):
- """Test base scraper string representation."""
-
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_dataset_123"
- PLATFORM_NAME = "testplatform"
-
- scraper = TestScraper(bearer_token="test_token_123456789")
- repr_str = repr(scraper)
-
- assert "testplatform" in repr_str.lower()
- assert "test_dataset_123" in repr_str
-
-
-class TestRegistryPattern:
- """Test registry pattern and auto-discovery."""
-
- def test_register_decorator_works(self):
- """Test @register decorator adds scraper to registry."""
-
- @register("testplatform")
- class TestScraper(BaseWebScraper):
- DATASET_ID = "test_123"
- PLATFORM_NAME = "testplatform"
-
- # Should be in registry
- scraper_class = get_scraper_for("https://testplatform.com/page")
- assert scraper_class is TestScraper
-
- def test_get_scraper_for_amazon_url(self):
- """Test get_scraper_for returns AmazonScraper for Amazon URLs."""
- scraper_class = get_scraper_for("https://www.amazon.com/dp/B123")
- assert scraper_class is AmazonScraper
-
- def test_get_scraper_for_linkedin_url(self):
- """Test get_scraper_for returns LinkedInScraper for LinkedIn URLs."""
- scraper_class = get_scraper_for("https://linkedin.com/in/johndoe")
- assert scraper_class is LinkedInScraper
-
- def test_get_scraper_for_chatgpt_url(self):
- """Test get_scraper_for returns ChatGPTScraper for ChatGPT URLs."""
- scraper_class = get_scraper_for("https://chatgpt.com/c/abc123")
- assert scraper_class is ChatGPTScraper
-
- def test_get_scraper_for_unknown_domain_returns_none(self):
- """Test get_scraper_for returns None for unknown domains."""
- scraper_class = get_scraper_for("https://unknown-domain-xyz.com/page")
- assert scraper_class is None
-
- def test_get_registered_platforms(self):
- """Test get_registered_platforms returns all registered platforms."""
- platforms = get_registered_platforms()
-
- assert isinstance(platforms, list)
- assert "amazon" in platforms
- assert "linkedin" in platforms
- assert "chatgpt" in platforms
-
- def test_is_platform_supported_for_known_platform(self):
- """Test is_platform_supported returns True for known platforms."""
- assert is_platform_supported("https://amazon.com/dp/B123") is True
- assert is_platform_supported("https://linkedin.com/in/john") is True
-
- def test_is_platform_supported_for_unknown_platform(self):
- """Test is_platform_supported returns False for unknown platforms."""
- assert is_platform_supported("https://unknown.com/page") is False
-
-
-class TestAmazonScraper:
- """Test AmazonScraper platform-specific features."""
-
- def test_amazon_scraper_has_correct_attributes(self):
- """Test AmazonScraper has correct dataset ID and platform name."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "amazon"
- assert scraper.DATASET_ID == "gd_l7q7dkf244hwjntr0"
- assert scraper.MIN_POLL_TIMEOUT == 240
- assert scraper.COST_PER_RECORD == 0.001 # Uses DEFAULT_COST_PER_RECORD
-
- def test_amazon_scraper_has_products_method(self):
- """Test AmazonScraper has products search method (async-first API)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "products")
- assert callable(scraper.products)
-
- def test_amazon_scraper_has_reviews_method(self):
- """Test AmazonScraper has reviews method (async-first API)."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "reviews")
- assert callable(scraper.reviews)
-
- def test_amazon_scraper_registered_in_registry(self):
- """Test AmazonScraper is registered for 'amazon' domain."""
- scraper_class = get_scraper_for("https://amazon.com/dp/B123")
- assert scraper_class is AmazonScraper
-
-
-class TestLinkedInScraper:
- """Test LinkedInScraper platform-specific features."""
-
- def test_linkedin_scraper_has_correct_attributes(self):
- """Test LinkedInScraper has correct dataset IDs."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "linkedin"
- assert scraper.DATASET_ID.startswith("gd_") # People profiles
- assert hasattr(scraper, "DATASET_ID_COMPANIES")
- assert hasattr(scraper, "DATASET_ID_JOBS")
-
- def test_linkedin_scraper_has_profiles_method(self):
- """Test LinkedInScraper has profiles search method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "profiles")
- assert callable(scraper.profiles)
-
- def test_linkedin_scraper_has_companies_method(self):
- """Test LinkedInScraper has companies search method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "companies")
- assert callable(scraper.companies)
-
- def test_linkedin_scraper_has_jobs_method(self):
- """Test LinkedInScraper has jobs search method (async-first API)."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "jobs")
- assert callable(scraper.jobs)
-
- def test_linkedin_scraper_registered_in_registry(self):
- """Test LinkedInScraper is registered for 'linkedin' domain."""
- scraper_class = get_scraper_for("https://linkedin.com/in/john")
- assert scraper_class is LinkedInScraper
-
-
-class TestChatGPTScraper:
- """Test ChatGPTScraper platform-specific features."""
-
- def test_chatgpt_scraper_has_correct_attributes(self):
- """Test ChatGPTScraper has correct dataset ID."""
- scraper = ChatGPTScraper(bearer_token="test_token_123456789")
-
- assert scraper.PLATFORM_NAME == "chatgpt"
- assert scraper.DATASET_ID.startswith("gd_")
-
- def test_chatgpt_scraper_has_prompt_method(self):
- """Test ChatGPTScraper has prompt method (async-first API)."""
- scraper = ChatGPTScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "prompt")
- assert callable(scraper.prompt)
-
- def test_chatgpt_scraper_has_prompts_method(self):
- """Test ChatGPTScraper has prompts (batch) method (async-first API)."""
- scraper = ChatGPTScraper(bearer_token="test_token_123456789")
-
- assert hasattr(scraper, "prompts")
- assert callable(scraper.prompts)
-
- def test_chatgpt_scraper_scrape_raises_not_implemented(self):
- """Test ChatGPTScraper raises NotImplementedError for scrape()."""
- import asyncio
-
- scraper = ChatGPTScraper(bearer_token="test_token_123456789")
-
- async def test_scrape():
- with pytest.raises(NotImplementedError) as exc_info:
- await scraper.scrape("https://chatgpt.com/")
- assert "doesn't support URL-based scraping" in str(exc_info.value)
- assert "Use prompt()" in str(exc_info.value)
-
- asyncio.run(test_scrape())
-
- def test_chatgpt_scraper_registered_in_registry(self):
- """Test ChatGPTScraper is registered for 'chatgpt' domain."""
- scraper_class = get_scraper_for("https://chatgpt.com/c/123")
- assert scraper_class is ChatGPTScraper
-
-
-class TestScrapeVsSearchDistinction:
- """Test clear distinction between scrape and search methods."""
-
- def test_scrape_methods_are_url_based(self):
- """Test scrape() methods accept URLs."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- # scrape() should accept URL
- assert hasattr(scraper, "scrape")
- # Method signature should accept urls parameter
- import inspect
-
- sig = inspect.signature(scraper.scrape)
- assert "urls" in sig.parameters
-
- def test_search_methods_are_parameter_based(self):
- """Test search methods (discovery) accept keywords/parameters."""
- # Search methods are in search services, not scrapers
- # Scrapers are now URL-based only per API spec
-
- from brightdata.scrapers.linkedin import LinkedInSearchScraper
-
- linkedin_search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- import inspect
-
- # LinkedIn search jobs() should accept keyword (parameter-based discovery)
- jobs_sig = inspect.signature(linkedin_search.jobs)
- assert "keyword" in jobs_sig.parameters
-
- # LinkedIn search profiles() should accept first_name (parameter-based discovery)
- profiles_sig = inspect.signature(linkedin_search.profiles)
- assert "first_name" in profiles_sig.parameters
-
- # LinkedIn search posts() should accept url (parameter-based discovery)
- posts_sig = inspect.signature(linkedin_search.posts)
- assert "url" in posts_sig.parameters
-
- def test_all_platform_scrapers_have_scrape(self):
- """Test all platform scrapers have scrape() method."""
- scrapers = [
- AmazonScraper(bearer_token="test_token_123456789"),
- LinkedInScraper(bearer_token="test_token_123456789"),
- # ChatGPT is exception - it overrides to raise NotImplementedError
- ]
-
- for scraper in scrapers:
- assert hasattr(scraper, "scrape")
- assert callable(scraper.scrape)
-
- def test_platforms_have_all_methods(self):
- """Test all platforms have their methods (async-first API)."""
- amazon = AmazonScraper(bearer_token="test_token_123456789")
- linkedin = LinkedInScraper(bearer_token="test_token_123456789")
-
- # Amazon - all URL-based scrape methods
- assert hasattr(amazon, "products") and callable(amazon.products)
- assert hasattr(amazon, "reviews") and callable(amazon.reviews)
- assert hasattr(amazon, "sellers") and callable(amazon.sellers)
-
- # LinkedIn - URL-based scrape methods
- assert hasattr(linkedin, "posts") and callable(linkedin.posts)
- assert hasattr(linkedin, "jobs") and callable(linkedin.jobs)
- assert hasattr(linkedin, "profiles") and callable(linkedin.profiles)
- assert hasattr(linkedin, "companies") and callable(linkedin.companies)
-
-
-class TestClientIntegration:
- """Test scrapers integrate with BrightDataClient."""
-
- def test_scrapers_accessible_through_client(self):
- """Test scrapers are accessible through client.scrape namespace."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- # All scrapers should be accessible
- assert hasattr(client.scrape, "amazon")
- assert hasattr(client.scrape, "linkedin")
- assert hasattr(client.scrape, "chatgpt")
-
- def test_client_scraper_access_returns_correct_instances(self):
- """Test client returns correct scraper instances."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- amazon = client.scrape.amazon
- assert isinstance(amazon, AmazonScraper)
- assert amazon.PLATFORM_NAME == "amazon"
-
- linkedin = client.scrape.linkedin
- assert isinstance(linkedin, LinkedInScraper)
- assert linkedin.PLATFORM_NAME == "linkedin"
-
- chatgpt = client.scrape.chatgpt
- assert isinstance(chatgpt, ChatGPTScraper)
- assert chatgpt.PLATFORM_NAME == "chatgpt"
-
- def test_client_passes_token_to_scrapers(self):
- """Test client passes its token to scraper instances."""
- from brightdata import BrightDataClient
-
- token = "test_token_123456789"
- client = BrightDataClient(token=token)
-
- amazon = client.scrape.amazon
- assert amazon.bearer_token == token
-
-
-class TestInterfaceConsistency:
- """Test interface consistency across platforms."""
-
- def test_amazon_interface_matches_spec(self):
- """Test Amazon scraper matches interface specification."""
- scraper = AmazonScraper(bearer_token="test_token_123456789")
-
- # URL-based scraping
- assert hasattr(scraper, "scrape")
-
- # Parameter-based search
- assert hasattr(scraper, "products")
- assert hasattr(scraper, "reviews")
-
- def test_linkedin_interface_matches_spec(self):
- """Test LinkedIn scraper matches interface specification."""
- scraper = LinkedInScraper(bearer_token="test_token_123456789")
-
- # URL-based scraping
- assert hasattr(scraper, "scrape")
-
- # Parameter-based search
- assert hasattr(scraper, "profiles")
- assert hasattr(scraper, "companies")
- assert hasattr(scraper, "jobs")
-
- def test_chatgpt_interface_matches_spec(self):
- """Test ChatGPT scraper matches interface specification."""
- import asyncio
-
- scraper = ChatGPTScraper(bearer_token="test_token_123456789")
-
- # Prompt-based (ChatGPT specific)
- assert hasattr(scraper, "prompt")
- assert hasattr(scraper, "prompts")
-
- # scrape() should raise NotImplementedError (async method)
- async def test_scrape():
- with pytest.raises(NotImplementedError):
- await scraper.scrape("https://chatgpt.com/")
-
- asyncio.run(test_scrape())
-
-
-class TestPhilosophicalPrinciples:
- """Test scrapers follow philosophical principles."""
-
- def test_platforms_feel_familiar(self):
- """Test platforms have similar interfaces (familiarity)."""
- amazon = AmazonScraper(bearer_token="test_token_123456789")
- linkedin = LinkedInScraper(bearer_token="test_token_123456789")
-
- # Both should have scrape() method (async-first API)
- assert hasattr(amazon, "scrape")
- assert hasattr(linkedin, "scrape")
- assert callable(amazon.scrape)
- assert callable(linkedin.scrape)
-
- def test_scrape_vs_search_is_clear(self):
- """Test scrape vs search distinction is clear."""
- amazon = AmazonScraper(bearer_token="test_token_123456789")
-
- import inspect
-
- # Amazon products() is now URL-based scraping (not search)
- products_sig = inspect.signature(amazon.products)
- assert "url" in products_sig.parameters
- assert "sync" not in products_sig.parameters # sync parameter was removed
-
- # For search methods, check LinkedInSearchScraper
- from brightdata.scrapers.linkedin import LinkedInSearchScraper
-
- linkedin_search = LinkedInSearchScraper(bearer_token="test_token_123456789")
-
- # Search jobs() signature = parameter-based (has keyword, not url required)
- jobs_sig = inspect.signature(linkedin_search.jobs)
- assert "keyword" in jobs_sig.parameters
-
- def test_architecture_supports_future_auto_routing(self):
- """Test architecture is ready for future auto-routing."""
- # Registry pattern enables auto-routing
- amazon_url = "https://amazon.com/dp/B123"
- scraper_class = get_scraper_for(amazon_url)
-
- assert scraper_class is not None
- assert scraper_class is AmazonScraper
-
- # This enables future: client.scrape.auto(url)
- # The infrastructure is in place!
diff --git a/tests/unit/test_serp.py b/tests/unit/test_serp.py
deleted file mode 100644
index 53f5a92..0000000
--- a/tests/unit/test_serp.py
+++ /dev/null
@@ -1,507 +0,0 @@
-"""Unit tests for SERP service."""
-
-from brightdata.api.serp import (
- BaseSERPService,
- GoogleSERPService,
- BingSERPService,
- YandexSERPService,
-)
-
-
-class TestBaseSERPService:
- """Test base SERP service functionality."""
-
- def test_base_serp_has_search_engine_attribute(self):
- """Test base SERP service has SEARCH_ENGINE attribute."""
- assert hasattr(BaseSERPService, "SEARCH_ENGINE")
- assert hasattr(BaseSERPService, "ENDPOINT")
-
- def test_base_serp_has_search_methods(self):
- """Test base SERP service has search methods (async-first API)."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- assert hasattr(service, "search")
- assert callable(service.search)
-
- def test_base_serp_has_data_normalizer(self):
- """Test base SERP has data_normalizer."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- assert hasattr(service, "data_normalizer")
- assert hasattr(service.data_normalizer, "normalize")
- assert callable(service.data_normalizer.normalize)
-
-
-class TestGoogleSERPService:
- """Test Google SERP service."""
-
- def test_google_serp_has_correct_engine_name(self):
- """Test Google SERP service has correct search engine name."""
- assert GoogleSERPService.SEARCH_ENGINE == "google"
-
- def test_google_serp_build_search_url(self):
- """Test Google search URL building."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- url = service.url_builder.build(
- query="python tutorial",
- location="United States",
- language="en",
- device="desktop",
- num_results=10,
- )
-
- assert "google.com/search" in url
- assert "q=python+tutorial" in url or "q=python%20tutorial" in url
- assert "num=10" in url
- assert "hl=en" in url
- assert "gl=" in url # Location code
-
- def test_google_serp_url_encoding(self):
- """Test Google search query encoding."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- url = service.url_builder.build(
- query="python & javascript",
- location=None,
- language="en",
- device="desktop",
- num_results=10,
- )
-
- # Should encode special characters
- assert "google.com/search" in url
- assert "+" in url or "%20" in url # Space encoded
-
- def test_google_serp_location_parsing(self):
- """Test location name to country code parsing."""
- from brightdata.utils.location import LocationService, LocationFormat
-
- # Test country name mappings
- assert LocationService.parse_location("United States", LocationFormat.GOOGLE) == "us"
- assert LocationService.parse_location("United Kingdom", LocationFormat.GOOGLE) == "gb"
- assert LocationService.parse_location("Canada", LocationFormat.GOOGLE) == "ca"
-
- # Test direct codes
- assert LocationService.parse_location("US", LocationFormat.GOOGLE) == "us"
- assert LocationService.parse_location("GB", LocationFormat.GOOGLE) == "gb"
-
- def test_google_serp_normalize_data(self):
- """Test Google SERP data normalization."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- # Test with structured data
- raw_data = {
- "organic": [
- {
- "title": "Python Tutorial",
- "url": "https://python.org/tutorial",
- "description": "Learn Python",
- },
- {
- "title": "Advanced Python",
- "url": "https://example.com/advanced",
- "description": "Advanced topics",
- },
- ],
- "total_results": 1000000,
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- assert "results" in normalized
- assert len(normalized["results"]) == 2
- assert normalized["results"][0]["position"] == 1
- assert normalized["results"][0]["title"] == "Python Tutorial"
- assert normalized["results"][1]["position"] == 2
-
- def test_google_serp_normalize_empty_data(self):
- """Test Google SERP normalization with empty data."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- # Normalization is done via data_normalizer attribute
- normalized = service.data_normalizer.normalize({})
- assert "results" in normalized
- assert normalized["results"] == []
-
-
-class TestBingSERPService:
- """Test Bing SERP service."""
-
- def test_bing_serp_has_correct_engine_name(self):
- """Test Bing SERP service has correct search engine name."""
- assert BingSERPService.SEARCH_ENGINE == "bing"
-
- def test_bing_serp_build_search_url(self):
- """Test Bing search URL building."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = BingSERPService(engine)
-
- url = service.url_builder.build(
- query="python tutorial",
- location="United States",
- language="en",
- device="desktop",
- num_results=10,
- )
-
- assert "bing.com/search" in url
- assert "q=python" in url
- assert "count=10" in url
-
-
-class TestYandexSERPService:
- """Test Yandex SERP service."""
-
- def test_yandex_serp_has_correct_engine_name(self):
- """Test Yandex SERP service has correct search engine name."""
- assert YandexSERPService.SEARCH_ENGINE == "yandex"
-
- def test_yandex_serp_build_search_url(self):
- """Test Yandex search URL building."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = YandexSERPService(engine)
-
- url = service.url_builder.build(
- query="python tutorial",
- location="Russia",
- language="ru",
- device="desktop",
- num_results=10,
- )
-
- assert "yandex.com/search" in url
- assert "text=python" in url
- assert "numdoc=10" in url
-
-
-class TestSERPNormalization:
- """Test SERP data normalization across engines."""
-
- def test_normalized_results_have_position(self):
- """Test normalized results include ranking position."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [
- {"title": "Result 1", "url": "https://example1.com", "description": "Desc 1"},
- {"title": "Result 2", "url": "https://example2.com", "description": "Desc 2"},
- ]
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- # Each result should have position starting from 1
- for i, result in enumerate(normalized["results"], 1):
- assert result["position"] == i
-
- def test_normalized_results_have_required_fields(self):
- """Test normalized results have required fields."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [
- {"title": "Test", "url": "https://test.com", "description": "Test desc"},
- ]
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
- result = normalized["results"][0]
-
- # Required fields
- assert "position" in result
- assert "title" in result
- assert "url" in result
- assert "description" in result
-
-
-class TestClientIntegration:
- """Test SERP services integrate with BrightDataClient."""
-
- def test_search_service_accessible_through_client(self):
- """Test search service is accessible via client.search."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client, "search")
- assert client.search is not None
-
- def test_search_service_has_google_method(self):
- """Test search service has google() method (async-first API)."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.search, "google")
- assert callable(client.search.google)
-
- def test_search_service_has_bing_method(self):
- """Test search service has bing() method (async-first API)."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.search, "bing")
- assert callable(client.search.bing)
-
- def test_search_service_has_yandex_method(self):
- """Test search service has yandex() method (async-first API)."""
- from brightdata import BrightDataClient
-
- client = BrightDataClient(token="test_token_123456789")
-
- assert hasattr(client.search, "yandex")
- assert callable(client.search.yandex)
-
-
-class TestSERPInterfaceConsistency:
- """Test interface consistency across search engines."""
-
- def test_all_engines_have_same_signature(self):
- """Test all search engines have consistent method signatures."""
- from brightdata import BrightDataClient
- import inspect
-
- client = BrightDataClient(token="test_token_123456789")
-
- # Get signatures
- google_sig = inspect.signature(client.search.google)
- bing_sig = inspect.signature(client.search.bing)
- yandex_sig = inspect.signature(client.search.yandex)
-
- # All should have 'query' parameter
- assert "query" in google_sig.parameters
- assert "query" in bing_sig.parameters
- assert "query" in yandex_sig.parameters
-
- def test_all_engines_return_search_result(self):
- """Test all engines return SearchResult type."""
- from brightdata import BrightDataClient
- import inspect
-
- client = BrightDataClient(token="test_token_123456789")
-
- # Check return type hints if available (async-first API)
- google_sig = inspect.signature(client.search.google)
- # Return annotation should mention SearchResult or List[SearchResult]
- if google_sig.return_annotation != inspect.Signature.empty:
- assert "SearchResult" in str(google_sig.return_annotation)
-
-
-class TestPhilosophicalPrinciples:
- """Test SERP service follows philosophical principles."""
-
- def test_serp_data_normalized_across_engines(self):
- """Test SERP data is normalized for easy comparison."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
-
- # Same raw data structure
- raw_data = {
- "organic": [
- {"title": "Result", "url": "https://example.com", "description": "Desc"},
- ],
- "total_results": 1000,
- }
-
- # Both engines should normalize to same format
- google_service = GoogleSERPService(engine)
- google_normalized = google_service.data_normalizer.normalize(raw_data)
-
- # Normalized format should have:
- assert "results" in google_normalized
- assert "total_results" in google_normalized
- assert isinstance(google_normalized["results"], list)
-
- def test_search_engine_quirks_handled_transparently(self):
- """Test search engine specific quirks are abstracted away."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
-
- # Different engines have different URL patterns
- google = GoogleSERPService(engine)
- bing = BingSERPService(engine)
- yandex = YandexSERPService(engine)
-
- # But all build URLs transparently
- google_url = google.url_builder.build("test", None, "en", "desktop", 10)
- bing_url = bing.url_builder.build("test", None, "en", "desktop", 10)
- yandex_url = yandex.url_builder.build("test", None, "ru", "desktop", 10)
-
- # Each should have their engine's domain
- assert "google.com" in google_url
- assert "bing.com" in bing_url
- assert "yandex.com" in yandex_url
-
- # But query is present in all
- assert "test" in google_url
- assert "test" in bing_url
- assert "test" in yandex_url
-
- def test_results_include_ranking_position(self):
- """Test results include ranking position for competitive analysis."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [
- {"title": "First", "url": "https://1.com", "description": "D1"},
- {"title": "Second", "url": "https://2.com", "description": "D2"},
- {"title": "Third", "url": "https://3.com", "description": "D3"},
- ]
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- # Positions should be 1, 2, 3
- positions = [r["position"] for r in normalized["results"]]
- assert positions == [1, 2, 3]
-
-
-class TestSERPFeatureExtraction:
- """Test SERP feature detection and extraction."""
-
- def test_extract_featured_snippet(self):
- """Test extraction of featured snippet."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [],
- "featured_snippet": {
- "title": "What is Python?",
- "description": "Python is a programming language...",
- "url": "https://python.org",
- },
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- assert "featured_snippet" in normalized
- assert normalized["featured_snippet"]["title"] == "What is Python?"
-
- def test_extract_knowledge_panel(self):
- """Test extraction of knowledge panel."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [],
- "knowledge_panel": {
- "title": "Python",
- "type": "Programming Language",
- "description": "High-level programming language",
- },
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- assert "knowledge_panel" in normalized
- assert normalized["knowledge_panel"]["title"] == "Python"
-
- def test_extract_people_also_ask(self):
- """Test extraction of People Also Ask section."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- raw_data = {
- "organic": [],
- "people_also_ask": [
- {"question": "What is Python used for?", "answer": "..."},
- {"question": "Is Python easy to learn?", "answer": "..."},
- ],
- }
-
- normalized = service.data_normalizer.normalize(raw_data)
-
- assert "people_also_ask" in normalized
- assert len(normalized["people_also_ask"]) == 2
-
-
-class TestLocationLanguageSupport:
- """Test location and language-specific search support."""
-
- def test_google_supports_location(self):
- """Test Google search supports location parameter."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- url = service.url_builder.build(
- query="restaurants",
- location="New York",
- language="en",
- device="desktop",
- num_results=10,
- )
-
- # Should have location parameter
- assert "gl=" in url
-
- def test_google_supports_language(self):
- """Test Google search supports language parameter."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- url_en = service.url_builder.build("test", None, "en", "desktop", 10)
- url_es = service.url_builder.build("test", None, "es", "desktop", 10)
- url_fr = service.url_builder.build("test", None, "fr", "desktop", 10)
-
- assert "hl=en" in url_en
- assert "hl=es" in url_es
- assert "hl=fr" in url_fr
-
- def test_google_supports_device_types(self):
- """Test Google search supports device type parameter."""
- from brightdata.core.engine import AsyncEngine
-
- engine = AsyncEngine("test_token_123456789")
- service = GoogleSERPService(engine)
-
- service.url_builder.build("test", None, "en", "desktop", 10)
- url_mobile = service.url_builder.build("test", None, "en", "mobile", 10)
-
- # Mobile should have mobile-specific parameter
- assert "mobile" in url_mobile.lower() or "mobileaction" in url_mobile
diff --git a/tests/unit/test_ssl_helpers.py b/tests/unit/test_ssl_helpers.py
index 224db1b..3f2fe4f 100644
--- a/tests/unit/test_ssl_helpers.py
+++ b/tests/unit/test_ssl_helpers.py
@@ -1,226 +1,178 @@
-"""Unit tests for SSL error handling utilities."""
+"""Tests for utils/ssl_helpers.py — SSL error detection and messages."""
import ssl
from unittest.mock import Mock, patch
+
from brightdata.utils.ssl_helpers import is_macos, is_ssl_certificate_error, get_ssl_error_message
-class TestPlatformDetection:
- """Test platform detection utilities."""
+# ---------------------------------------------------------------------------
+# Platform detection
+# ---------------------------------------------------------------------------
- def test_is_macos_returns_boolean(self):
- """Test is_macos returns a boolean."""
- result = is_macos()
- assert isinstance(result, bool)
+
+class TestPlatformDetection:
+ def test_returns_boolean(self):
+ assert isinstance(is_macos(), bool)
@patch("sys.platform", "darwin")
- def test_is_macos_true_on_darwin(self):
- """Test is_macos returns True on darwin platform."""
- result = is_macos()
- assert result is True
+ def test_true_on_darwin(self):
+ assert is_macos() is True
@patch("sys.platform", "linux")
- def test_is_macos_false_on_linux(self):
- """Test is_macos returns False on linux."""
- result = is_macos()
- assert result is False
+ def test_false_on_linux(self):
+ assert is_macos() is False
@patch("sys.platform", "win32")
- def test_is_macos_false_on_windows(self):
- """Test is_macos returns False on Windows."""
- result = is_macos()
- assert result is False
+ def test_false_on_windows(self):
+ assert is_macos() is False
+
+
+# ---------------------------------------------------------------------------
+# SSL certificate error detection
+# ---------------------------------------------------------------------------
class TestSSLCertificateErrorDetection:
- """Test SSL certificate error detection."""
+ def test_ssl_error_detected(self):
+ assert is_ssl_certificate_error(ssl.SSLError("certificate verify failed")) is True
- def test_ssl_error_is_detected(self):
- """Test SSL errors are detected."""
- error = ssl.SSLError("certificate verify failed")
- assert is_ssl_certificate_error(error) is True
+ def test_oserror_with_ssl_keywords_detected(self):
+ assert is_ssl_certificate_error(OSError("SSL certificate verification failed")) is True
- def test_oserror_with_ssl_keywords_is_detected(self):
- """Test OSError with SSL keywords is detected."""
- error = OSError("SSL certificate verification failed")
- assert is_ssl_certificate_error(error) is True
+ def test_oserror_with_certificate_keyword_detected(self):
+ assert is_ssl_certificate_error(OSError("unable to get local issuer certificate")) is True
- def test_oserror_with_certificate_keyword_is_detected(self):
- """Test OSError with 'certificate' keyword is detected."""
- error = OSError("unable to get local issuer certificate")
- assert is_ssl_certificate_error(error) is True
+ def test_generic_exception_with_ssl_message_detected(self):
+ assert is_ssl_certificate_error(Exception("[SSL: CERTIFICATE_VERIFY_FAILED]")) is True
- def test_generic_exception_with_ssl_message_is_detected(self):
- """Test generic exception with SSL message is detected."""
- error = Exception("[SSL: CERTIFICATE_VERIFY_FAILED]")
- assert is_ssl_certificate_error(error) is True
+ def test_certificate_verify_failed_detected(self):
+ assert is_ssl_certificate_error(Exception("certificate verify failed")) is True
- def test_exception_with_certificate_verify_failed(self):
- """Test exception with 'certificate verify failed' is detected."""
- error = Exception("certificate verify failed")
- assert is_ssl_certificate_error(error) is True
+ def test_non_ssl_error_not_detected(self):
+ assert is_ssl_certificate_error(ValueError("Invalid value")) is False
- def test_non_ssl_error_is_not_detected(self):
- """Test non-SSL errors are not detected."""
- error = ValueError("Invalid value")
- assert is_ssl_certificate_error(error) is False
+ def test_connection_error_without_ssl_not_detected(self):
+ assert is_ssl_certificate_error(ConnectionError("Connection refused")) is False
- def test_connection_error_without_ssl_is_not_detected(self):
- """Test connection errors without SSL keywords are not detected."""
- error = ConnectionError("Connection refused")
- assert is_ssl_certificate_error(error) is False
+ def test_timeout_error_not_detected(self):
+ assert is_ssl_certificate_error(TimeoutError("Operation timed out")) is False
- def test_timeout_error_is_not_detected(self):
- """Test timeout errors are not detected as SSL errors."""
- error = TimeoutError("Operation timed out")
- assert is_ssl_certificate_error(error) is False
+# ---------------------------------------------------------------------------
+# SSL error messages
+# ---------------------------------------------------------------------------
-class TestSSLErrorMessage:
- """Test SSL error message generation."""
+class TestSSLErrorMessage:
@patch("brightdata.utils.ssl_helpers.is_macos", return_value=True)
- def test_macos_error_message_includes_platform_specific_fixes(self, mock_is_macos):
- """Test macOS error message includes platform-specific fixes."""
- error = ssl.SSLError("certificate verify failed")
- message = get_ssl_error_message(error)
+ def test_macos_includes_platform_specific_fixes(self, _):
+ message = get_ssl_error_message(ssl.SSLError("certificate verify failed"))
- # Should include base message
assert "SSL certificate verification failed" in message
assert "macOS" in message
-
- # Should include macOS-specific fixes
assert "Install Certificates.command" in message
assert "Homebrew" in message
assert "certifi" in message
assert "SSL_CERT_FILE" in message
@patch("brightdata.utils.ssl_helpers.is_macos", return_value=False)
- def test_non_macos_error_message_excludes_macos_specific_fixes(self, mock_is_macos):
- """Test non-macOS error message excludes macOS-specific fixes."""
- error = ssl.SSLError("certificate verify failed")
- message = get_ssl_error_message(error)
+ def test_non_macos_excludes_macos_fixes(self, _):
+ message = get_ssl_error_message(ssl.SSLError("certificate verify failed"))
- # Should include base message
assert "SSL certificate verification failed" in message
-
- # Should NOT include macOS-specific fixes
assert "Install Certificates.command" not in message
assert "Homebrew" not in message
-
- # Should include generic fixes
assert "certifi" in message
assert "SSL_CERT_FILE" in message
- def test_error_message_includes_original_error(self):
- """Test error message includes original error."""
- error = ssl.SSLError("specific error details")
- message = get_ssl_error_message(error)
-
+ def test_includes_original_error(self):
+ message = get_ssl_error_message(ssl.SSLError("specific error details"))
assert "Original error:" in message
assert "specific error details" in message
- def test_error_message_includes_fix_instructions(self):
- """Test error message includes fix instructions."""
- error = ssl.SSLError("certificate verify failed")
- message = get_ssl_error_message(error)
-
- # Should include pip install command
+ def test_includes_fix_instructions(self):
+ message = get_ssl_error_message(ssl.SSLError("certificate verify failed"))
assert "pip install" in message
assert "certifi" in message
-
- # Should include SSL_CERT_FILE command
assert "export SSL_CERT_FILE" in message
assert "python -m certifi" in message
- def test_error_message_includes_documentation_link(self):
- """Test error message includes documentation link."""
- error = ssl.SSLError("certificate verify failed")
- message = get_ssl_error_message(error)
-
- # Should include link to troubleshooting docs
+ def test_includes_documentation_link(self):
+ message = get_ssl_error_message(ssl.SSLError("certificate verify failed"))
assert "docs/troubleshooting" in message or "troubleshooting.md" in message
-class TestSSLErrorMessageFormats:
- """Test SSL error message handles different error formats."""
+# ---------------------------------------------------------------------------
+# Different error formats
+# ---------------------------------------------------------------------------
- def test_ssl_error_with_detailed_message(self):
- """Test handling of SSL error with detailed message."""
+
+class TestSSLErrorFormats:
+ def test_detailed_ssl_error(self):
error = ssl.SSLError(
- "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate"
+ "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: "
+ "unable to get local issuer certificate"
)
message = get_ssl_error_message(error)
-
assert message is not None
- assert len(message) > 0
assert "SSL certificate verification failed" in message
def test_oserror_with_ssl_context(self):
- """Test handling of OSError with SSL context."""
- error = OSError(1, "SSL: certificate verify failed")
- message = get_ssl_error_message(error)
-
+ message = get_ssl_error_message(OSError(1, "SSL: certificate verify failed"))
assert message is not None
assert len(message) > 0
def test_generic_exception_with_ssl_message(self):
- """Test handling of generic exception with SSL message."""
- error = Exception("SSL certificate problem: unable to get local issuer certificate")
- message = get_ssl_error_message(error)
-
+ message = get_ssl_error_message(
+ Exception("SSL certificate problem: unable to get local issuer certificate")
+ )
assert message is not None
assert len(message) > 0
-class TestSSLErrorDetectionEdgeCases:
- """Test SSL error detection edge cases."""
+# ---------------------------------------------------------------------------
+# Edge cases
+# ---------------------------------------------------------------------------
+
+class TestSSLEdgeCases:
def test_empty_error_message(self):
- """Test handling of error with empty message."""
- error = Exception("")
- assert is_ssl_certificate_error(error) is False
+ assert is_ssl_certificate_error(Exception("")) is False
- def test_none_error_message(self):
- """Test handling of error with None message."""
+ def test_none_error_message_does_not_crash(self):
error = Mock()
error.__str__ = Mock(return_value=None)
- # Should not crash - handle None return gracefully
try:
result = is_ssl_certificate_error(error)
assert isinstance(result, bool)
except (TypeError, AttributeError):
- # If __str__ returns None, we should handle it gracefully
- # This is acceptable behavior - function should not crash
- assert True
-
- def test_ssl_keyword_case_insensitive(self):
- """Test SSL keyword detection is case-insensitive."""
- error1 = Exception("SSL CERTIFICATE VERIFY FAILED")
- error2 = Exception("ssl certificate verify failed")
- error3 = Exception("Ssl Certificate Verify Failed")
-
- assert is_ssl_certificate_error(error1) is True
- assert is_ssl_certificate_error(error2) is True
- assert is_ssl_certificate_error(error3) is True
-
- def test_partial_ssl_keyword_match(self):
- """Test partial SSL keyword matches are detected."""
- # "certificate" keyword alone should match
- error = Exception("invalid certificate")
- assert is_ssl_certificate_error(error) is True
+ pass # acceptable — function should not crash
+
+ def test_case_insensitive_detection(self):
+ assert is_ssl_certificate_error(Exception("SSL CERTIFICATE VERIFY FAILED")) is True
+ assert is_ssl_certificate_error(Exception("ssl certificate verify failed")) is True
+ assert is_ssl_certificate_error(Exception("Ssl Certificate Verify Failed")) is True
+
+ def test_partial_keyword_match(self):
+ assert is_ssl_certificate_error(Exception("invalid certificate")) is True
+
+ def test_keyword_in_middle_of_message(self):
+ assert (
+ is_ssl_certificate_error(
+ Exception("Connection failed due to SSL certificate verification error")
+ )
+ is True
+ )
- def test_ssl_error_in_middle_of_message(self):
- """Test SSL keywords in middle of message are detected."""
- error = Exception("Connection failed due to SSL certificate verification error")
- assert is_ssl_certificate_error(error) is True
+# ---------------------------------------------------------------------------
+# Integration
+# ---------------------------------------------------------------------------
-class TestSSLHelperIntegration:
- """Test SSL helper integration scenarios."""
- def test_can_identify_and_format_common_ssl_errors(self):
- """Test can identify and format common SSL error scenarios."""
+class TestSSLIntegration:
+ def test_common_ssl_errors_identified_and_formatted(self):
common_errors = [
ssl.SSLError("certificate verify failed"),
Exception("[SSL: CERTIFICATE_VERIFY_FAILED]"),
@@ -229,16 +181,12 @@ def test_can_identify_and_format_common_ssl_errors(self):
]
for error in common_errors:
- # Should be identified as SSL error
assert is_ssl_certificate_error(error) is True
-
- # Should generate helpful message
message = get_ssl_error_message(error)
- assert len(message) > 100 # Should be substantial
+ assert len(message) > 100
assert "certifi" in message.lower()
- def test_non_ssl_errors_dont_trigger_ssl_handling(self):
- """Test non-SSL errors don't trigger SSL handling."""
+ def test_non_ssl_errors_not_flagged(self):
non_ssl_errors = [
ValueError("Invalid parameter"),
KeyError("missing_key"),
diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py
deleted file mode 100644
index 5bf955b..0000000
--- a/tests/unit/test_validation.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Unit tests for validation."""
diff --git a/tests/unit/test_zone_manager.py b/tests/unit/test_zone_manager.py
index 04e48c9..0709c96 100644
--- a/tests/unit/test_zone_manager.py
+++ b/tests/unit/test_zone_manager.py
@@ -1,121 +1,97 @@
-"""Unit tests for ZoneManager."""
+"""Tests for core/zone_manager.py — Zone CRUD and ensure operations."""
import pytest
-from unittest.mock import MagicMock
+
from brightdata.core.zone_manager import ZoneManager
from brightdata.exceptions.errors import ZoneError, AuthenticationError
+from tests.conftest import MockResponse, MockContextManager
-class MockResponse:
- """Mock aiohttp response for testing."""
-
- def __init__(self, status: int, json_data=None, text_data=""):
- self.status = status
- self._json_data = json_data
- self._text_data = text_data
-
- async def json(self):
- return self._json_data
-
- async def text(self):
- return self._text_data
-
- async def __aenter__(self):
- return self
-
- async def __aexit__(self, exc_type, exc_val, exc_tb):
- pass
-
-
-@pytest.fixture
-def mock_engine():
- """Create a mock engine for testing."""
- engine = MagicMock()
- return engine
+# ---------------------------------------------------------------------------
+# List Zones
+# ---------------------------------------------------------------------------
-class TestZoneManagerListZones:
- """Tests for listing zones."""
+class TestListZones:
@pytest.mark.asyncio
- async def test_list_zones_success(self, mock_engine):
- """Test successful zone listing."""
+ async def test_returns_zones_list(self, mock_engine):
zones_data = [{"name": "zone1", "type": "unblocker"}, {"name": "zone2", "type": "serp"}]
- mock_engine.get.return_value = MockResponse(200, json_data=zones_data)
+ mock_engine.get.return_value = MockContextManager(MockResponse(200, json_data=zones_data))
- zone_manager = ZoneManager(mock_engine)
- zones = await zone_manager.list_zones()
+ zm = ZoneManager(mock_engine)
+ zones = await zm.list_zones()
assert zones == zones_data
mock_engine.get.assert_called_once_with("/zone/get_active_zones")
@pytest.mark.asyncio
- async def test_list_zones_empty(self, mock_engine):
- """Test listing zones when none exist."""
- mock_engine.get.return_value = MockResponse(200, json_data=[])
+ async def test_returns_empty_list_when_none(self, mock_engine):
+ mock_engine.get.return_value = MockContextManager(MockResponse(200, json_data=[]))
- zone_manager = ZoneManager(mock_engine)
- zones = await zone_manager.list_zones()
+ zm = ZoneManager(mock_engine)
+ zones = await zm.list_zones()
assert zones == []
@pytest.mark.asyncio
- async def test_list_zones_null_response(self, mock_engine):
- """Test listing zones when API returns null."""
- mock_engine.get.return_value = MockResponse(200, json_data=None)
+ async def test_returns_empty_list_on_null_response(self, mock_engine):
+ mock_engine.get.return_value = MockContextManager(MockResponse(200, json_data=None))
- zone_manager = ZoneManager(mock_engine)
- zones = await zone_manager.list_zones()
+ zm = ZoneManager(mock_engine)
+ zones = await zm.list_zones()
assert zones == []
@pytest.mark.asyncio
- async def test_list_zones_auth_error_401(self, mock_engine):
- """Test listing zones with 401 authentication error."""
- mock_engine.get.return_value = MockResponse(401, text_data="Invalid token")
+ async def test_401_raises_authentication_error(self, mock_engine):
+ mock_engine.get.return_value = MockContextManager(
+ MockResponse(401, text_data="Invalid token")
+ )
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(AuthenticationError) as exc_info:
- await zone_manager.list_zones()
+ await zm.list_zones()
assert "401" in str(exc_info.value)
assert "Invalid token" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_list_zones_auth_error_403(self, mock_engine):
- """Test listing zones with 403 forbidden error."""
- mock_engine.get.return_value = MockResponse(403, text_data="Forbidden")
+ async def test_403_raises_authentication_error(self, mock_engine):
+ mock_engine.get.return_value = MockContextManager(MockResponse(403, text_data="Forbidden"))
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(AuthenticationError) as exc_info:
- await zone_manager.list_zones()
+ await zm.list_zones()
assert "403" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_list_zones_api_error(self, mock_engine):
- """Test listing zones with general API error."""
- mock_engine.get.return_value = MockResponse(500, text_data="Internal server error")
+ async def test_500_raises_zone_error(self, mock_engine):
+ mock_engine.get.return_value = MockContextManager(
+ MockResponse(500, text_data="Internal server error")
+ )
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(ZoneError) as exc_info:
- await zone_manager.list_zones()
+ await zm.list_zones()
assert "500" in str(exc_info.value)
-class TestZoneManagerCreateZone:
- """Tests for zone creation."""
+# ---------------------------------------------------------------------------
+# Create Zone
+# ---------------------------------------------------------------------------
+
+class TestCreateZone:
@pytest.mark.asyncio
- async def test_create_unblocker_zone_success(self, mock_engine):
- """Test creating an unblocker zone successfully."""
- mock_engine.post.return_value = MockResponse(201)
+ async def test_creates_unblocker_zone(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager._create_zone("test_unblocker", "unblocker")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("test_unblocker", "unblocker")
- # Verify the POST was called with correct payload
mock_engine.post.assert_called_once()
call_args = mock_engine.post.call_args
assert call_args[0][0] == "/zone"
@@ -125,239 +101,218 @@ async def test_create_unblocker_zone_success(self, mock_engine):
assert payload["plan"]["type"] == "unblocker"
@pytest.mark.asyncio
- async def test_create_serp_zone_success(self, mock_engine):
- """Test creating a SERP zone successfully."""
- mock_engine.post.return_value = MockResponse(200)
+ async def test_creates_serp_zone(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(MockResponse(200))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager._create_zone("test_serp", "serp")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("test_serp", "serp")
- # Verify the POST was called with correct payload
- call_args = mock_engine.post.call_args
- payload = call_args[1]["json_data"]
+ payload = mock_engine.post.call_args[1]["json_data"]
assert payload["zone"]["name"] == "test_serp"
assert payload["zone"]["type"] == "serp"
assert payload["plan"]["type"] == "unblocker"
assert payload["plan"]["serp"] is True
@pytest.mark.asyncio
- async def test_create_browser_zone_success(self, mock_engine):
- """Test creating a browser zone successfully."""
- mock_engine.post.return_value = MockResponse(201)
+ async def test_creates_browser_zone(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager._create_zone("test_browser", "browser")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("test_browser", "browser")
- call_args = mock_engine.post.call_args
- payload = call_args[1]["json_data"]
+ payload = mock_engine.post.call_args[1]["json_data"]
assert payload["zone"]["name"] == "test_browser"
assert payload["zone"]["type"] == "browser"
assert payload["plan"]["type"] == "browser"
@pytest.mark.asyncio
- async def test_create_zone_already_exists_409(self, mock_engine):
- """Test creating a zone that already exists (409)."""
- mock_engine.post.return_value = MockResponse(409, text_data="Conflict")
+ async def test_409_conflict_does_not_raise(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(MockResponse(409, text_data="Conflict"))
- zone_manager = ZoneManager(mock_engine)
- # Should not raise an exception
- await zone_manager._create_zone("existing_zone", "unblocker")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("existing_zone", "unblocker") # should not raise
@pytest.mark.asyncio
- async def test_create_zone_already_exists_message(self, mock_engine):
- """Test creating a zone with duplicate message in response."""
- mock_engine.post.return_value = MockResponse(400, text_data="Zone already exists")
+ async def test_already_exists_message_does_not_raise(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(
+ MockResponse(400, text_data="Zone already exists")
+ )
- zone_manager = ZoneManager(mock_engine)
- # Should not raise an exception
- await zone_manager._create_zone("existing_zone", "unblocker")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("existing_zone", "unblocker") # should not raise
@pytest.mark.asyncio
- async def test_create_zone_duplicate_message(self, mock_engine):
- """Test creating a zone with duplicate name error."""
- mock_engine.post.return_value = MockResponse(400, text_data="Duplicate zone name")
+ async def test_duplicate_name_message_does_not_raise(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(
+ MockResponse(400, text_data="Duplicate zone name")
+ )
- zone_manager = ZoneManager(mock_engine)
- # Should not raise an exception
- await zone_manager._create_zone("duplicate_zone", "unblocker")
+ zm = ZoneManager(mock_engine)
+ await zm._create_zone("duplicate_zone", "unblocker") # should not raise
@pytest.mark.asyncio
- async def test_create_zone_auth_error_401(self, mock_engine):
- """Test zone creation with authentication error."""
- mock_engine.post.return_value = MockResponse(401, text_data="Unauthorized")
+ async def test_401_raises_authentication_error(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(
+ MockResponse(401, text_data="Unauthorized")
+ )
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(AuthenticationError) as exc_info:
- await zone_manager._create_zone("test_zone", "unblocker")
+ await zm._create_zone("test_zone", "unblocker")
assert "401" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_create_zone_auth_error_403(self, mock_engine):
- """Test zone creation with forbidden error."""
- mock_engine.post.return_value = MockResponse(403, text_data="Forbidden")
+ async def test_403_raises_authentication_error(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(MockResponse(403, text_data="Forbidden"))
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(AuthenticationError) as exc_info:
- await zone_manager._create_zone("test_zone", "unblocker")
+ await zm._create_zone("test_zone", "unblocker")
assert "403" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_create_zone_bad_request(self, mock_engine):
- """Test zone creation with bad request error."""
- mock_engine.post.return_value = MockResponse(400, text_data="Invalid zone configuration")
+ async def test_400_bad_request_raises_zone_error(self, mock_engine):
+ mock_engine.post.return_value = MockContextManager(
+ MockResponse(400, text_data="Invalid zone configuration")
+ )
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
with pytest.raises(ZoneError) as exc_info:
- await zone_manager._create_zone("test_zone", "unblocker")
+ await zm._create_zone("test_zone", "unblocker")
assert "400" in str(exc_info.value)
assert "Invalid zone configuration" in str(exc_info.value)
-class TestZoneManagerEnsureZones:
- """Tests for ensuring zones exist."""
+# ---------------------------------------------------------------------------
+# Ensure Required Zones
+# ---------------------------------------------------------------------------
+
+class TestEnsureRequiredZones:
@pytest.mark.asyncio
- async def test_ensure_zones_all_exist(self, mock_engine):
- """Test ensuring zones when all already exist."""
+ async def test_skips_creation_when_all_exist(self, mock_engine):
zones_data = [
{"name": "sdk_unlocker", "type": "unblocker"},
{"name": "sdk_serp", "type": "serp"},
]
- mock_engine.get.return_value = MockResponse(200, json_data=zones_data)
+ mock_engine.get.return_value = MockContextManager(MockResponse(200, json_data=zones_data))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager.ensure_required_zones(
- web_unlocker_zone="sdk_unlocker", serp_zone="sdk_serp"
- )
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="sdk_unlocker", serp_zone="sdk_serp")
- # Should only call GET to list zones, not POST to create
mock_engine.get.assert_called()
mock_engine.post.assert_not_called()
@pytest.mark.asyncio
- async def test_ensure_zones_create_missing(self, mock_engine):
- """Test ensuring zones when some need to be created."""
- # First call: existing zones (empty)
- # After creation: zones exist
+ async def test_creates_missing_zones(self, mock_engine):
mock_engine.get.side_effect = [
- MockResponse(200, json_data=[]), # Initial list
- MockResponse(
- 200,
- json_data=[ # Verification list
- {"name": "sdk_unlocker", "type": "unblocker"},
- {"name": "sdk_serp", "type": "serp"},
- ],
+ MockContextManager(MockResponse(200, json_data=[])),
+ MockContextManager(
+ MockResponse(
+ 200,
+ json_data=[
+ {"name": "sdk_unlocker", "type": "unblocker"},
+ {"name": "sdk_serp", "type": "serp"},
+ ],
+ )
),
]
- mock_engine.post.return_value = MockResponse(201)
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager.ensure_required_zones(
- web_unlocker_zone="sdk_unlocker", serp_zone="sdk_serp"
- )
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="sdk_unlocker", serp_zone="sdk_serp")
- # Should create both zones
assert mock_engine.post.call_count == 2
@pytest.mark.asyncio
- async def test_ensure_zones_only_web_unlocker(self, mock_engine):
- """Test ensuring only web unlocker zone."""
+ async def test_creates_only_web_unlocker(self, mock_engine):
mock_engine.get.side_effect = [
- MockResponse(200, json_data=[]),
- MockResponse(200, json_data=[{"name": "sdk_unlocker"}]),
+ MockContextManager(MockResponse(200, json_data=[])),
+ MockContextManager(MockResponse(200, json_data=[{"name": "sdk_unlocker"}])),
]
- mock_engine.post.return_value = MockResponse(201)
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager.ensure_required_zones(web_unlocker_zone="sdk_unlocker")
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="sdk_unlocker")
- # Should only create web unlocker zone
assert mock_engine.post.call_count == 1
@pytest.mark.asyncio
- async def test_ensure_zones_with_browser(self, mock_engine):
- """Test ensuring unblocker and SERP zones (browser zones NOT auto-created)."""
+ async def test_creates_unblocker_and_serp(self, mock_engine):
mock_engine.get.side_effect = [
- MockResponse(200, json_data=[]),
- MockResponse(200, json_data=[{"name": "sdk_unlocker"}, {"name": "sdk_serp"}]),
+ MockContextManager(MockResponse(200, json_data=[])),
+ MockContextManager(
+ MockResponse(
+ 200,
+ json_data=[
+ {"name": "sdk_unlocker"},
+ {"name": "sdk_serp"},
+ ],
+ )
+ ),
]
- mock_engine.post.return_value = MockResponse(201)
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- await zone_manager.ensure_required_zones(
- web_unlocker_zone="sdk_unlocker",
- serp_zone="sdk_serp",
- browser_zone="sdk_browser", # This is passed but NOT created (by design)
- )
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="sdk_unlocker", serp_zone="sdk_serp")
- # Should only create unblocker + SERP zones (browser zones require manual setup)
assert mock_engine.post.call_count == 2
@pytest.mark.asyncio
- async def test_ensure_zones_verification_fails(self, mock_engine, caplog):
- """Test zone creation when verification fails (logs warning but doesn't raise)."""
- # Zones never appear in verification (max_attempts = 5, so need 6 total responses)
+ async def test_verification_failure_logs_warning(self, mock_engine, caplog):
mock_engine.get.side_effect = [
- MockResponse(200, json_data=[]), # Initial list
- MockResponse(200, json_data=[]), # Verification attempt 1
- MockResponse(200, json_data=[]), # Verification attempt 2
- MockResponse(200, json_data=[]), # Verification attempt 3
- MockResponse(200, json_data=[]), # Verification attempt 4
- MockResponse(200, json_data=[]), # Verification attempt 5 (final)
+ MockContextManager(MockResponse(200, json_data=[])), # initial list
+ MockContextManager(MockResponse(200, json_data=[])), # verify 1
+ MockContextManager(MockResponse(200, json_data=[])), # verify 2
+ MockContextManager(MockResponse(200, json_data=[])), # verify 3
+ MockContextManager(MockResponse(200, json_data=[])), # verify 4
+ MockContextManager(MockResponse(200, json_data=[])), # verify 5
]
- mock_engine.post.return_value = MockResponse(201)
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- zone_manager = ZoneManager(mock_engine)
- # Verification failure should log warning but NOT raise exception
- await zone_manager.ensure_required_zones(web_unlocker_zone="sdk_unlocker")
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="sdk_unlocker")
- # Should have logged warning about verification failure
- assert any("Zone verification failed" in record.message for record in caplog.records)
+ assert any("Zone verification failed" in r.message for r in caplog.records)
-class TestZoneManagerIntegration:
- """Integration-style tests for ZoneManager."""
+# ---------------------------------------------------------------------------
+# Integration-style
+# ---------------------------------------------------------------------------
+
+class TestZoneManagerIntegration:
@pytest.mark.asyncio
- async def test_full_workflow_no_zones_to_create(self, mock_engine):
- """Test full workflow when zones already exist."""
+ async def test_full_workflow_no_creation_needed(self, mock_engine):
zones_data = [{"name": "my_zone", "type": "unblocker", "status": "active"}]
- mock_engine.get.return_value = MockResponse(200, json_data=zones_data)
+ mock_engine.get.return_value = MockContextManager(MockResponse(200, json_data=zones_data))
- zone_manager = ZoneManager(mock_engine)
+ zm = ZoneManager(mock_engine)
- # List zones
- zones = await zone_manager.list_zones()
+ zones = await zm.list_zones()
assert len(zones) == 1
assert zones[0]["name"] == "my_zone"
- # Ensure zones (should not create any)
- await zone_manager.ensure_required_zones(web_unlocker_zone="my_zone")
+ await zm.ensure_required_zones(web_unlocker_zone="my_zone")
mock_engine.post.assert_not_called()
@pytest.mark.asyncio
- async def test_full_workflow_create_zones(self, mock_engine):
- """Test full workflow creating new zones."""
+ async def test_full_workflow_creates_then_lists(self, mock_engine):
zones_after = [{"name": "new_zone", "type": "unblocker"}]
mock_engine.get.side_effect = [
- MockResponse(200, json_data=[]), # Initial list (empty)
- MockResponse(200, json_data=zones_after), # After creation (verification)
- MockResponse(200, json_data=zones_after), # List zones again
+ MockContextManager(MockResponse(200, json_data=[])),
+ MockContextManager(MockResponse(200, json_data=zones_after)),
+ MockContextManager(MockResponse(200, json_data=zones_after)),
]
- mock_engine.post.return_value = MockResponse(201)
-
- zone_manager = ZoneManager(mock_engine)
-
- # Ensure zones (should create)
- await zone_manager.ensure_required_zones(web_unlocker_zone="new_zone")
+ mock_engine.post.return_value = MockContextManager(MockResponse(201))
- # Verify zone was created
+ zm = ZoneManager(mock_engine)
+ await zm.ensure_required_zones(web_unlocker_zone="new_zone")
assert mock_engine.post.call_count == 1
- # List zones again
- zones = await zone_manager.list_zones()
+ zones = await zm.list_zones()
assert len(zones) == 1
assert zones[0]["name"] == "new_zone"