# add headers and cookies headers = {} cookies = {} query = """ query GetModularFeed($searchParams: [SearchParam], $debug: Boolean = false) {\n modularFeed(params: $searchParams, debug: $debug) {\n analyticsData {\n requestId\n searchPerformedEventUniqueId\n searchSessionId\n __typename\n }\n categoryInfo {\n categoryId\n isForcedCategory\n __typename\n }\n feedAdditions\n filters {\n ...modularFilterNumericRange\n ...modularFilterSelectionList\n __typename\n }\n legacyFeedOptions {\n ...legacyFeedOptionListSelection\n ...legacyFeedOptionNumericRange\n __typename\n }\n looseTiles {\n ...modularTileBanner\n ...modularTileBingAd\n ...modularTileGoogleDisplayAd\n ...modularTileJob\n ...modularTileEmptyState\n ...modularTileListing\n ...modularTileLocalDisplayAd\n ...modularTileSearchAlert\n ...modularTileSellerAd\n ...modularModuleTileAdsPostXAd\n __typename\n }\n modules {\n ...modularGridModule\n __typename\n }\n pageCursor\n query {\n ...modularQueryInfo\n __typename\n }\n requestTimeMetadata {\n resolverComputationTimeSeconds\n serviceRequestTimeSeconds\n totalResolverTimeSeconds\n __typename\n }\n searchAlert {\n alertId\n alertStatus\n searchAlertCount\n __typename\n }\n personalizationPath\n debugInformation @include(if: $debug) {\n rankedListings {\n listingId\n attributes {\n key\n value\n __typename\n }\n __typename\n }\n lastViewedItems {\n listingId\n attributes {\n key\n value\n __typename\n }\n __typename\n }\n categoryAffinities {\n affinity\n count\n decay\n affinityOwner\n __typename\n }\n rankingStats {\n key\n value\n __typename\n }\n __typename\n }\n __typename\n }\n}\n\nfragment modularFilterNumericRange on ModularFeedNumericRangeFilter {\n isExpandedHighlight\n lowerBound {\n ...modularFilterNumericRangeBound\n __typename\n }\n shortcutLabel\n shortcutRank\n subTitle\n targetName\n title\n type\n upperBound {\n ...modularFilterNumericRangeBound\n __typename\n }\n __typename\n}\n\nfragment modularFilterNumericRangeBound on ModularFeedNumericRangeFilterNumericRangeBound {\n label\n limit\n placeholderText\n targetName\n value\n __typename\n}\n\nfragment modularFilterSelectionList on ModularFeedSelectionListFilter {\n targetName\n title\n subTitle\n shortcutLabel\n shortcutRank\n type\n isExpandedHighlight\n options {\n ...modularFilterSelectionListOption\n __typename\n }\n __typename\n}\n\nfragment modularFilterSelectionListOption on ModularFeedSelectionListFilterOption {\n isDefault\n isSelected\n label\n subLabel\n value\n __typename\n}\n\nfragment legacyFeedOptionListSelection on FeedOptionListSelection {\n label\n labelShort\n name\n options {\n default\n label\n labelShort\n selected\n subLabel\n value\n __typename\n }\n position\n queryParam\n type\n __typename\n}\n\nfragment legacyFeedOptionNumericRange on FeedOptionNumericRange {\n label\n labelShort\n leftQueryParam\n lowerBound\n name\n options {\n currentValue\n label\n textHint\n __typename\n }\n position\n rightQueryParam\n type\n units\n upperBound\n __typename\n}\n\nfragment modularTileBanner on ModularFeedTileBanner {\n tileId\n tileType\n title\n __typename\n}\n\nfragment modularTileBingAd on ModularFeedTileBingAd {\n tileId\n bingAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n adSettings {\n repeatClickRefractoryPeriodMillis\n collapsible\n __typename\n }\n bingClientId\n clickFeedbackUrl\n clickReturnUrl\n contentUrl\n deepLinkEnabled\n experimentDataHash\n image {\n height\n url\n width\n __typename\n }\n impressionFeedbackUrl\n impressionUrls\n viewableImpressionUrls\n installmentInfo {\n amount\n description\n downPayment\n __typename\n }\n itemName\n lowPrice\n price\n searchId\n sellerName\n templateFields {\n key\n value\n __typename\n }\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n tileId\n googleDisplayAd {\n ouAdId\n additionalSizes\n adExperimentId\n adHeight\n adNetwork\n adPage\n adRequestId\n adTileType\n adWidth\n adaptive\n channel\n clickFeedbackUrl\n clientId\n contentUrl\n customTargeting {\n key\n values\n __typename\n }\n displayAdType\n errorDrawable {\n actionPath\n listImage {\n height\n url\n width\n __typename\n }\n __typename\n }\n experimentDataHash\n formatIds\n impressionFeedbackUrl\n personalizationProperties {\n key\n values\n __typename\n }\n prebidConfigs {\n key\n values {\n timeout\n tamSlotUUID\n liftoffPlacementIDs\n nimbusPriceMapping\n adPosition\n __typename\n }\n __typename\n }\n renderLocation\n searchId\n searchQuery\n templateId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileJob on ModularFeedTileJob {\n tileId\n tileType\n job {\n address {\n city\n state\n zipcode\n __typename\n }\n companyName\n datePosted\n image {\n height\n url\n width\n __typename\n }\n industry\n jobId\n jobListingUrl\n jobOwnerId\n pills {\n text\n type\n __typename\n }\n title\n apply {\n method\n value\n __typename\n }\n wageDisplayValue\n provider\n __typename\n }\n __typename\n}\n\nfragment modularTileEmptyState on ModularFeedTileEmptyState {\n tileId\n tileType\n title\n description\n iconType\n __typename\n}\n\nfragment modularTileListing on ModularFeedTileListing {\n tileId\n listing {\n ...modularListing\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularListing on ModularFeedListing {\n listingId\n conditionText\n flags\n image {\n height\n url\n width\n __typename\n }\n isFirmPrice\n locationName\n price\n title\n vehicleMiles\n __typename\n}\n\nfragment modularTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n tileId\n localDisplayAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n advertiserId\n businessName\n callToAction\n callToActionType\n clickFeedbackUrl\n contentUrl\n experimentDataHash\n headline\n image {\n height\n url\n width\n __typename\n }\n impressionFeedbackUrl\n searchId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileSearchAlert on ModularFeedTileSearchAlert {\n tileId\n tileType\n title\n __typename\n}\n\nfragment modularTileSellerAd on ModularFeedTileSellerAd {\n tileId\n listing {\n ...modularListing\n __typename\n }\n sellerAd {\n ouAdId\n adId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n clickFeedbackUrl\n experimentDataHash\n impressionFeedbackUrl\n searchId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularModuleTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n ...modularTileAdsPostXAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n tileId\n adsPostXAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n clickFeedbackUrl\n experimentDataHash\n impressionFeedbackUrl\n searchId\n offer {\n beacons {\n noThanksClick\n close\n __typename\n }\n title\n description\n clickUrl\n image\n pixel\n ctaYes\n ctaNo\n perkswallLabel\n perkswallUrl\n __typename\n }\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularGridModule on ModularFeedModuleGrid {\n moduleId\n collection\n formFactor\n grid {\n actionPath\n tiles {\n ...modularModuleTileBingAd\n ...modularModuleTileGoogleDisplayAd\n ...modularModuleTileListing\n ...modularModuleTileLocalDisplayAd\n ...modularModuleTileSellerAd\n __typename\n }\n __typename\n }\n moduleType\n rank\n rowIndex\n searchId\n subTitle\n title\n infoActionPath\n feedIndex\n __typename\n}\n\nfragment modularModuleTileBingAd on ModularFeedTileBingAd {\n ...modularTileBingAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n ...modularTileGoogleDisplayAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileListing on ModularFeedTileListing {\n ...modularTileListing\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n ...modularTileLocalDisplayAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileSellerAd on ModularFeedTileSellerAd {\n ...modularTileSellerAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularQueryInfo on ModularFeedQueryInfo {\n appliedQuery\n decisionType\n originalQuery\n suggestedQuery\n __typename\n}\n """ random_delay = await random_interval() all_listings = [] page_cursor = None current_page = 0 search_session_id = '579cfbd4-193d-44d8-8ba5-b5cefeabe928' # Get search_session_id from the website's headers or may be it's in payload # Loop to fetch multiple pages while current_page < max_pages: # print(f"Scrolled {count+1} pages out of {max_pages}") # Prepare search parameters - always include the latest page_cursor search_params = [ {'key': 'platform', 'value': 'web'}, {'key': 'lon', 'value': '-77.1995'}, {'key': 'lat', 'value': '38.696'}, {'key': 'zipcode', 'value': '33503'}, {'key': 'DISTANCE', 'value': '200'}, # 200 in miles {'key': 'cid', 'value': str(category_id)}, # category id is based on the category of the listings, please refer to the network tools of the browser {'key': 'limit', 'value': '50'}, {'key': 'searchSessionId', 'value': search_session_id} ] # Page Cursor is generate after each scroll if page_cursor: search_params.append({'key': 'page_cursor', 'value': page_cursor}) json_data = { 'operationName': 'GetModularFeed', 'variables': { 'debug': False, 'searchParams': search_params, }, 'query': query, } try: # I have create a make_requests function sepearately, please use your own preferred requests module. I have used curl_cffi one. response = await make_requests( 'post', 'https://offerup.com/api/graphql', headers=headers, cookies=cookies, json=json_data, ) data = response.json() # Check if we have valid data if 'data' not in data or 'modularFeed' not in data['data'] or 'looseTiles' not in data['data']['modularFeed']: print(f"Invalid or empty response on page {current_page + 1}") break # Get the listings from this page page_listings = [] for tile in data['data']['modularFeed']['looseTiles']: # Check if this is a listing tile (not an ad or other content) if 'listing' in tile: print(tile['listing']['title']) listing_info = { 'title': tile['listing']['title'], # 'id': tile['listing']['listingId'], 'url': f"https://offerup.com/item/detail/{tile['listing']['listingId']}", 'price': tile['listing']['price'], # 'location': tile['listing']['locationName'] } page_listings.append(listing_info) # Add these listings to our main list all_listings.extend(page_listings) await asyncio.sleep(random_delay) print('--------------------------------------------------') print(f"Page {current_page + 1}: Found {len(page_listings)} listings") print('--------------------------------------------------\n') # Get the cursor for the next page new_cursor = data['data']['modularFeed'].get('pageCursor') # If we have no new cursor or it's the same as the previous one, we're done if not new_cursor or new_cursor == page_cursor: print("No more pages available") break # Update the cursor for the next request page_cursor = new_cursor current_page += 1 except Exception as e: print(f"Error fetching data: {str(e)}") break return all_listings