# add headers and cookies
headers = {}
cookies = {}
query = """
query GetModularFeed($searchParams: [SearchParam], $debug: Boolean = false) {\n modularFeed(params: $searchParams, debug: $debug) {\n analyticsData {\n requestId\n searchPerformedEventUniqueId\n searchSessionId\n __typename\n }\n categoryInfo {\n categoryId\n isForcedCategory\n __typename\n }\n feedAdditions\n filters {\n ...modularFilterNumericRange\n ...modularFilterSelectionList\n __typename\n }\n legacyFeedOptions {\n ...legacyFeedOptionListSelection\n ...legacyFeedOptionNumericRange\n __typename\n }\n looseTiles {\n ...modularTileBanner\n ...modularTileBingAd\n ...modularTileGoogleDisplayAd\n ...modularTileJob\n ...modularTileEmptyState\n ...modularTileListing\n ...modularTileLocalDisplayAd\n ...modularTileSearchAlert\n ...modularTileSellerAd\n ...modularModuleTileAdsPostXAd\n __typename\n }\n modules {\n ...modularGridModule\n __typename\n }\n pageCursor\n query {\n ...modularQueryInfo\n __typename\n }\n requestTimeMetadata {\n resolverComputationTimeSeconds\n serviceRequestTimeSeconds\n totalResolverTimeSeconds\n __typename\n }\n searchAlert {\n alertId\n alertStatus\n searchAlertCount\n __typename\n }\n personalizationPath\n debugInformation @include(if: $debug) {\n rankedListings {\n listingId\n attributes {\n key\n value\n __typename\n }\n __typename\n }\n lastViewedItems {\n listingId\n attributes {\n key\n value\n __typename\n }\n __typename\n }\n categoryAffinities {\n affinity\n count\n decay\n affinityOwner\n __typename\n }\n rankingStats {\n key\n value\n __typename\n }\n __typename\n }\n __typename\n }\n}\n\nfragment modularFilterNumericRange on ModularFeedNumericRangeFilter {\n isExpandedHighlight\n lowerBound {\n ...modularFilterNumericRangeBound\n __typename\n }\n shortcutLabel\n shortcutRank\n subTitle\n targetName\n title\n type\n upperBound {\n ...modularFilterNumericRangeBound\n __typename\n }\n __typename\n}\n\nfragment modularFilterNumericRangeBound on ModularFeedNumericRangeFilterNumericRangeBound {\n label\n limit\n placeholderText\n targetName\n value\n __typename\n}\n\nfragment modularFilterSelectionList on ModularFeedSelectionListFilter {\n targetName\n title\n subTitle\n shortcutLabel\n shortcutRank\n type\n isExpandedHighlight\n options {\n ...modularFilterSelectionListOption\n __typename\n }\n __typename\n}\n\nfragment modularFilterSelectionListOption on ModularFeedSelectionListFilterOption {\n isDefault\n isSelected\n label\n subLabel\n value\n __typename\n}\n\nfragment legacyFeedOptionListSelection on FeedOptionListSelection {\n label\n labelShort\n name\n options {\n default\n label\n labelShort\n selected\n subLabel\n value\n __typename\n }\n position\n queryParam\n type\n __typename\n}\n\nfragment legacyFeedOptionNumericRange on FeedOptionNumericRange {\n label\n labelShort\n leftQueryParam\n lowerBound\n name\n options {\n currentValue\n label\n textHint\n __typename\n }\n position\n rightQueryParam\n type\n units\n upperBound\n __typename\n}\n\nfragment modularTileBanner on ModularFeedTileBanner {\n tileId\n tileType\n title\n __typename\n}\n\nfragment modularTileBingAd on ModularFeedTileBingAd {\n tileId\n bingAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n adSettings {\n repeatClickRefractoryPeriodMillis\n collapsible\n __typename\n }\n bingClientId\n clickFeedbackUrl\n clickReturnUrl\n contentUrl\n deepLinkEnabled\n experimentDataHash\n image {\n height\n url\n width\n __typename\n }\n impressionFeedbackUrl\n impressionUrls\n viewableImpressionUrls\n installmentInfo {\n amount\n description\n downPayment\n __typename\n }\n itemName\n lowPrice\n price\n searchId\n sellerName\n templateFields {\n key\n value\n __typename\n }\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n tileId\n googleDisplayAd {\n ouAdId\n additionalSizes\n adExperimentId\n adHeight\n adNetwork\n adPage\n adRequestId\n adTileType\n adWidth\n adaptive\n channel\n clickFeedbackUrl\n clientId\n contentUrl\n customTargeting {\n key\n values\n __typename\n }\n displayAdType\n errorDrawable {\n actionPath\n listImage {\n height\n url\n width\n __typename\n }\n __typename\n }\n experimentDataHash\n formatIds\n impressionFeedbackUrl\n personalizationProperties {\n key\n values\n __typename\n }\n prebidConfigs {\n key\n values {\n timeout\n tamSlotUUID\n liftoffPlacementIDs\n nimbusPriceMapping\n adPosition\n __typename\n }\n __typename\n }\n renderLocation\n searchId\n searchQuery\n templateId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileJob on ModularFeedTileJob {\n tileId\n tileType\n job {\n address {\n city\n state\n zipcode\n __typename\n }\n companyName\n datePosted\n image {\n height\n url\n width\n __typename\n }\n industry\n jobId\n jobListingUrl\n jobOwnerId\n pills {\n text\n type\n __typename\n }\n title\n apply {\n method\n value\n __typename\n }\n wageDisplayValue\n provider\n __typename\n }\n __typename\n}\n\nfragment modularTileEmptyState on ModularFeedTileEmptyState {\n tileId\n tileType\n title\n description\n iconType\n __typename\n}\n\nfragment modularTileListing on ModularFeedTileListing {\n tileId\n listing {\n ...modularListing\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularListing on ModularFeedListing {\n listingId\n conditionText\n flags\n image {\n height\n url\n width\n __typename\n }\n isFirmPrice\n locationName\n price\n title\n vehicleMiles\n __typename\n}\n\nfragment modularTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n tileId\n localDisplayAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n advertiserId\n businessName\n callToAction\n callToActionType\n clickFeedbackUrl\n contentUrl\n experimentDataHash\n headline\n image {\n height\n url\n width\n __typename\n }\n impressionFeedbackUrl\n searchId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularTileSearchAlert on ModularFeedTileSearchAlert {\n tileId\n tileType\n title\n __typename\n}\n\nfragment modularTileSellerAd on ModularFeedTileSellerAd {\n tileId\n listing {\n ...modularListing\n __typename\n }\n sellerAd {\n ouAdId\n adId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n clickFeedbackUrl\n experimentDataHash\n impressionFeedbackUrl\n searchId\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularModuleTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n ...modularTileAdsPostXAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularTileAdsPostXAd on ModularFeedTileAdsPostXAd {\n tileId\n adsPostXAd {\n ouAdId\n adExperimentId\n adNetwork\n adRequestId\n adTileType\n clickFeedbackUrl\n experimentDataHash\n impressionFeedbackUrl\n searchId\n offer {\n beacons {\n noThanksClick\n close\n __typename\n }\n title\n description\n clickUrl\n image\n pixel\n ctaYes\n ctaNo\n perkswallLabel\n perkswallUrl\n __typename\n }\n __typename\n }\n tileType\n __typename\n}\n\nfragment modularGridModule on ModularFeedModuleGrid {\n moduleId\n collection\n formFactor\n grid {\n actionPath\n tiles {\n ...modularModuleTileBingAd\n ...modularModuleTileGoogleDisplayAd\n ...modularModuleTileListing\n ...modularModuleTileLocalDisplayAd\n ...modularModuleTileSellerAd\n __typename\n }\n __typename\n }\n moduleType\n rank\n rowIndex\n searchId\n subTitle\n title\n infoActionPath\n feedIndex\n __typename\n}\n\nfragment modularModuleTileBingAd on ModularFeedTileBingAd {\n ...modularTileBingAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileGoogleDisplayAd on ModularFeedTileGoogleDisplayAd {\n ...modularTileGoogleDisplayAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileListing on ModularFeedTileListing {\n ...modularTileListing\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileLocalDisplayAd on ModularFeedTileLocalDisplayAd {\n ...modularTileLocalDisplayAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularModuleTileSellerAd on ModularFeedTileSellerAd {\n ...modularTileSellerAd\n moduleId\n moduleRank\n moduleType\n __typename\n}\n\nfragment modularQueryInfo on ModularFeedQueryInfo {\n appliedQuery\n decisionType\n originalQuery\n suggestedQuery\n __typename\n}\n
"""
random_delay = await random_interval()
all_listings = []
page_cursor = None
current_page = 0
search_session_id = '579cfbd4-193d-44d8-8ba5-b5cefeabe928' # Get search_session_id from the website's headers or may be it's in payload
# Loop to fetch multiple pages
while current_page < max_pages:
# print(f"Scrolled {count+1} pages out of {max_pages}")
# Prepare search parameters - always include the latest page_cursor
search_params = [
{'key': 'platform', 'value': 'web'},
{'key': 'lon', 'value': '-77.1995'},
{'key': 'lat', 'value': '38.696'},
{'key': 'zipcode', 'value': '33503'},
{'key': 'DISTANCE', 'value': '200'}, # 200 in miles
{'key': 'cid', 'value': str(category_id)}, # category id is based on the category of the listings, please refer to the network tools of the browser
{'key': 'limit', 'value': '50'},
{'key': 'searchSessionId', 'value': search_session_id}
]
# Page Cursor is generate after each scroll
if page_cursor:
search_params.append({'key': 'page_cursor', 'value': page_cursor})
json_data = {
'operationName': 'GetModularFeed',
'variables': {
'debug': False,
'searchParams': search_params,
},
'query': query,
}
try:
# I have create a make_requests function sepearately, please use your own preferred requests module. I have used curl_cffi one.
response = await make_requests(
'post',
'https://offerup.com/api/graphql',
headers=headers,
cookies=cookies,
json=json_data,
)
data = response.json()
# Check if we have valid data
if 'data' not in data or 'modularFeed' not in data['data'] or 'looseTiles' not in data['data']['modularFeed']:
print(f"Invalid or empty response on page {current_page + 1}")
break
# Get the listings from this page
page_listings = []
for tile in data['data']['modularFeed']['looseTiles']:
# Check if this is a listing tile (not an ad or other content)
if 'listing' in tile:
print(tile['listing']['title'])
listing_info = {
'title': tile['listing']['title'],
# 'id': tile['listing']['listingId'],
'url': f"https://offerup.com/item/detail/{tile['listing']['listingId']}",
'price': tile['listing']['price'],
# 'location': tile['listing']['locationName']
}
page_listings.append(listing_info)
# Add these listings to our main list
all_listings.extend(page_listings)
await asyncio.sleep(random_delay)
print('--------------------------------------------------')
print(f"Page {current_page + 1}: Found {len(page_listings)} listings")
print('--------------------------------------------------\n')
# Get the cursor for the next page
new_cursor = data['data']['modularFeed'].get('pageCursor')
# If we have no new cursor or it's the same as the previous one, we're done
if not new_cursor or new_cursor == page_cursor:
print("No more pages available")
break
# Update the cursor for the next request
page_cursor = new_cursor
current_page += 1
except Exception as e:
print(f"Error fetching data: {str(e)}")
break
return all_listings
