python – Why 400 response status code when send POST?

Webscraping is a sensitive subject, and you should always respect the robots.txt file! The robot.txt file clearly states: Disallow: /sa/, hence probably the downvote you had.

For illustrative purposes, I will show how to retrieve the desired data.

1.) your form_data (in my case payload) was an awfully formatted dictionary with lists in it and so forth. As far as I am aware of Requests library can’t process this kind of format. Requests library only accepts simple dictionaries, and this way is also recommended to format them:
Firefox_browser example

2.) Your post URL can be copied completely from POST
POST url

3.) data=form_data should actually be json=form_data as your POST request will return a json formatted file.

Final remark is that the background database Algolia runs on Cloudflare, so it might be advised to use cloudscraper instead of requests library.

import requests

payload={"requests":[{"indexName":"prd_products_suggest","params":"highlightPreTag=%3Cstrong%3E&highlightPostTag=%3C%2Fstrong%3E&query=6970995781939&hitsPerPage=36&clickAnalytics=true&analyticsTags=%5B%22main%22%2C%22desktop%22%5D&ruleContexts=%5B%22ovr%22%2C%22desktop%22%2C%22t1%22%5D&facetingAfterDistinct=true&getRankingInfo=true&page=0&maxValuesPerFacet=10&facets=%5B%22manufacturer%22%2C%22offer.price%22%2C%22scegliPer%22%2C%22offer.shopType%22%2C%22reviews.avgRatingInt%22%2C%22navigation.lvl0%2Cnavigation.lvl1%2Cnavigation.lvl2%2Cnavigation.lvl3%22%5D&tagFilters="},{"indexName":"prd_products_suggest_b","params":"query=6970995781939&hitsPerPage=10&clickAnalytics=true&analyticsTags=%5B%22car_offerte_oggi%22%2C%22desktop%22%5D&ruleContexts=%5B%22ovr%22%2C%22car_offerte_oggi%22%2C%22desktop%22%5D&getRankingInfo=true&page=0&maxValuesPerFacet=10&minProximity=2&facetFilters=%5B%5D&facets=%5B%22manufacturer%22%2C%22offer.price%22%2C%22scegliPer%22%2C%22offer.shopType%22%2C%22reviews.avgRatingInt%22%2C%22navigation.lvl0%2Cnavigation.lvl1%2Cnavigation.lvl2%2Cnavigation.lvl3%22%5D&tagFilters="}]}

headers = {
"Host": "cgrd9wlxe4-dsn.algolia.net",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"content-type": "application/x-www-form-urlencoded",
"Content-Length": "1098",
"Origin": "https://www.eprice.it",
"DNT": "1",
"Connection": "keep-alive",
"Referer": "https://www.eprice.it/",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "cross-site",}

response = requests.post(
    url="https://cgrd9wlxe4-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia for JavaScript (4.11.0); Browser (lite)&x-algolia-api-key=e9c9895532cb88b620f96f3e6617c00f&x-algolia-application-id=CGRD9WLXE4",
    headers=headers,
     json=payload)

print(response.text)
{"results":[{"hits":[{"codArt":210757504,"sku":13940675,"codVen":"ROBOROCKS5MAXNERO","manufacturer":"XIAOMI","image":"/504/210757504/xiaomi-roborock-s5-max-robot-aspirapolvere-e-lavapavimenti-black.jpg","title":"Roborock S5 MAX nero robot aspirapolvere","reviews":{},"attributes":{},"attributes_hidden":{},"attributes_filter":{},"navigation":{"lvl0":"Elettrodomestici","lvl1":"Elettrodomestici > Pulizia Casa","lvl2":"Elettrodomestici > Pulizia Casa > Aspirapolvere","lvl3":"Elettrodomestici > Pulizia Casa > Aspirapolvere > Robot Aspirapolvere","slugs":["/elettrodomestici/pulizia-casa/aspirapolvere/robot-aspirapolvere","/elettrodomestici/pulizia-casa/aspirapolvere"]},"funzioneSpeciale":[],"flags":{"image":1,"description":1,"accessory":0,"refurbished":0,"bestseller":0,"bestPrice":1,"bbWinner":1,"sempreNuovo":0,"promo":0,"folliaGiorno":0,"stock":0,"fastShipping":0,"stockRemoto":0,"offerteDiOggi":0,"tasso0":0},"seoUrl":"/XIAOMI-Roborock-S5-MAX-nero-robot-aspirapolvere-ROBOROCKS5MAXNERO/d-13940675?shopid=10513","offer":{"shopId":10513,"shopType":"Marketplace","shopName":"AOC CENTER","quantity":1,"quantity_norm":100,"price":378.0,"price_norm10":3800,"price_norm100":400,"id":216682550},"scegliPer":[],"objectID":"210757504-MP","_highlightResult":{"title":{"value":"Roborock S5 MAX nero robot aspirapolvere","matchLevel":"none","matchedWords":[]}},"_rankingInfo":{"nbTypos":0,"firstMatchedWord":6000,"proximityDistance":0,"userScore":1,"geoDistance":0,"geoPrecision":1,"nbExactWords":1,"words":1,"filters":0}}],"nbHits":1,"page":0,"nbPages":1,"hitsPerPage":36,"facets":{"offer.price":{"378.0":1},"manufacturer":{"XIAOMI":1},"offer.shopType":{"Marketplace":1}},"facets_stats":{"offer.price":{"min":378,"max":378,"avg":378,"sum":378}},"exhaustiveFacetsCount":true,"exhaustiveFacetValues":false,"exhaustiveNbHits":true,"exhaustiveTypo":true,"query":"6970995781939","queryAfterRemoval":"6970995781939","params":"highlightPreTag=%3Cstrong%3E&highlightPostTag=%3C%2Fstrong%3E&query=6970995781939&hitsPerPage=36&clickAnalytics=true&analyticsTags=%5B%22main%22%2C%22desktop%22%5D&ruleContexts=%5B%22ovr%22%2C%22desktop%22%2C%22t1%22%5D&facetingAfterDistinct=true&getRankingInfo=true&page=0&maxValuesPerFacet=10&facets=%5B%22manufacturer%22%2C%22offer.price%22%2C%22scegliPer%22%2C%22offer.shopType%22%2C%22reviews.avgRatingInt%22%2C%22navigation.lvl0%2Cnavigation.lvl1%2Cnavigation.lvl2%2Cnavigation.lvl3%22%5D&tagFilters=&sumOrFiltersScores=true","index":"prd_products_suggest","queryID":"31487729ee9f180e6ac4849f82239e3f","serverUsed":"v8-de-1.algolia.net","indexUsed":"prd_products_suggest","parsedQuery":"6970995781939","timeoutCounts":false,"timeoutHits":false,"appliedRules":[{"objectID":"qr-1582840142298-680"}],"processingTimeMS":12},{"hits":[],"nbHits":0,"page":0,"nbPages":0,"hitsPerPage":10,"facets":{},"exhaustiveFacetsCount":false,"exhaustiveFacetValues":false,"exhaustiveNbHits":true,"exhaustiveTypo":true,"query":"6970995781939","queryAfterRemoval":"6970995781939","params":"query=6970995781939&hitsPerPage=10&clickAnalytics=true&analyticsTags=%5B%22car_offerte_oggi%22%2C%22desktop%22%5D&ruleContexts=%5B%22ovr%22%2C%22car_offerte_oggi%22%2C%22desktop%22%5D&getRankingInfo=true&page=0&maxValuesPerFacet=10&minProximity=2&facetFilters=%5B%5D&facets=%5B%22manufacturer%22%2C%22offer.price%22%2C%22scegliPer%22%2C%22offer.shopType%22%2C%22reviews.avgRatingInt%22%2C%22navigation.lvl0%2Cnavigation.lvl1%2Cnavigation.lvl2%2Cnavigation.lvl3%22%5D&tagFilters=&restrictSearchableAttributes=%5B%22search.manufacturer%22%2C%22search.tags%22%2C%22search.cat3%22%2C%22search.slug2_inv%22%2C%22attributes.serie%22%2C%22search.attributes_filter%22%2C%22search.title_comb%22%2C%22search.titleSlave%22%2C%22search.cat2%22%2C%22search.attributes%22%2C%22codArt%22%2C%22codVen%22%2C%22sku%22%2C%22ean%22%5D&minProximity=2&filters=offer.shopId%3D0+AND+flags.stock%3D1&optionalFilters=flags.offerteDiOggi%3A1%3Cscore%3D10%3E+OR+flags.folliaGiorno%3A1%3Cscore%3D3%3E+OR+promo.type%3ASAR%3Cscore%3D2%3E+OR+offer.quantity_norm%3A700%3Cscore%3D7%3E+OR+offer.quantity_norm%3A600%3Cscore%3D6%3E+OR+offer.quantity_norm%3A500%3Cscore%3D5%3E+OR+offer.quantity_norm%3A400%3Cscore%3D4%3E+OR+offer.quantity_norm%3A300%3Cscore%3D3%3E+OR+offer.quantity_norm%3A200%3Cscore%3D2%3E+OR+offer.quantity_norm%3A100%3Cscore%3D2%3E&sumOrFiltersScores=true","index":"prd_products_suggest_b","queryID":"4895f386f739bf7fc271730225d687c1","serverUsed":"v8-de-1.algolia.net","indexUsed":"prd_products_suggest_b","parsedQuery":"6970995781939","timeoutCounts":false,"timeoutHits":false,"appliedRules":[{"objectID":"qr-1583436915990-3590"},{"objectID":"qr-1582840142298-680"}],"processingTimeMS":15}]}

Leave a Comment