List Crawlers

curl --request GET \
  --url https://api.lobstr.io/v1/crawlers \
  --header 'Authorization: <authorization>'

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/crawlers', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/crawlers",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/crawlers"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/crawlers")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "total_results": 123,
  "limit": 123,
  "page": 123,
  "total_pages": 123,
  "result_from": 123,
  "result_to": 123,
  "data": [
    {}
  ],
  "next": {},
  "previous": {},
  "data[].id": "<string>",
  "data[].name": "<string>",
  "data[].slug": "<string>",
  "data[].is_public": true,
  "data[].is_available": true,
  "data[].is_premium": true,
  "data[].has_issues": true,
  "data[].max_concurrency": 123,
  "data[].credits_per_row": 123,
  "data[].account": {},
  "data[].icon": "<string>"
}

GET

crawlers

List Crawlers

curl --request GET \
  --url https://api.lobstr.io/v1/crawlers \
  --header 'Authorization: <authorization>'

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/crawlers', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/crawlers",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/crawlers"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/crawlers")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "total_results": 123,
  "limit": 123,
  "page": 123,
  "total_pages": 123,
  "result_from": 123,
  "result_to": 123,
  "data": [
    {}
  ],
  "next": {},
  "previous": {},
  "data[].id": "<string>",
  "data[].name": "<string>",
  "data[].slug": "<string>",
  "data[].is_public": true,
  "data[].is_available": true,
  "data[].is_premium": true,
  "data[].has_issues": true,
  "data[].max_concurrency": 123,
  "data[].credits_per_row": 123,
  "data[].account": {},
  "data[].icon": "<string>"
}

This endpoint returns a paginated list of all available crawlers on the lobstr.io platform. Each crawler is designed to extract specific types of data from different sources. Use this endpoint to discover which crawlers are available for your data collection needs before creating squids or adding tasks.

Headers

string

required

Your API authentication token. Value: Token YOUR_API_KEY

string

default:"application/json"

Content type of the request. Value: application/json

Response Structure

integer

Total number of crawlers available. Example: 56

integer

Number of results per page. Example: 50

integer

Current page number. Example: 1

integer

Total number of pages available. Example: 1

integer

Starting index of results in current page. Example: 1

integer

Ending index of results in current page. Example: 53

array

Array of crawler objects with comprehensive details

string | null

Full URL to the next page (null if on last page). Example: null

string | null

Full URL to the previous page (null if on first page). Example: null

Crawler Object Fields

string

Unique identifier (hash) for the crawler. Example: "4734d096159ef05210e0e1677e8be823"

string

Human-readable name of the crawler. Example: "Google Maps Search Export"

string

URL-friendly identifier. Example: "google-maps-search-export"

boolean

Whether the crawler is publicly available. Example: true

boolean

Whether the crawler is currently available for use (false if paused). Example: true

boolean

Requires premium subscription or extra credits. Example: false

boolean

Currently experiencing issues (runs paused until fixed). Example: false

integer

Maximum number of concurrent tasks allowed for this crawler. Example: 20

number

Credits charged per result row. Example: 0.5

object | null

Required account sync details (null if no authentication needed). Example: null

string

Base64-encoded SVG icon for UI display. Example: "data:image/svg+xml;base64,PHN2ZyB4bWxucz..."

Cache the list of crawlers locally as it doesn’t change frequently. This reduces unnecessary API calls and improves your application’s performance.

Each crawler has a unique hash identifier that you’ll use when creating squids or configuring data collection tasks.

Check the crawler parameters endpoint to see what configuration options are available for each crawler before starting your scraping operations.

Available Crawlers

Quick reference of all public crawlers and their IDs. Use the Crawler ID as the squid_hash when creating a squid or as the crawler_id when querying by crawler.

Crawler Name	Crawler ID
Amazon Reviews Scraper	`787abfafd394be3b12ee2ac7d14303fe`
AutoScout24 Listings Scraper	`0a09add2a3c455037a972db81b1c0219`
Bien’Ici Search Export	`53317cbe332e408c5c46372a18e4fc76`
Facebook Business Page Scraper	`c80e36ed76bfae3c332e8983971403d7`
Facebook Marketplace Scraper	`640e3d2f62769800fdb05f957f3a2415`
Facebook Pages Search Export	`c47dfc2beba9fdc2472d3b3321b06b66`
Facebook Private Group Posts Scraper	`3eea12bc7d80cf05b6fc38eaf9f28f77`
Facebook Public Group Posts Scraper	`9b0bb2e04c6f7c475a9da9bb20530dac`
Google Maps Leads Scraper	`4734d096159ef05210e0e1677e8be823`
Google Maps Reviews Scraper	`e625ce52e98a6d4d1b53ef36dff7dacb`
Google News Search Export	`06faf01896336c01ffb5e96d13dd406c`
Google Search Scraper	`f588385d11c48f8af51560e561077238`
Idealista Listings Search Export	`5f4b269e29e601e09fa50f4155b3970e`
Immoweb Listings & Phones Search Export	`9bca6a0a4cc5237b37798f8640dce087`
Instagram Followers Scraper	`5575499b33df1f165e72b29a12892a9e`
Instagram Post Comments Scraper	`438eabd1f513b6773600c166ad3ddbb4`
Instagram Post Scraper	`d25fddae52eb78eb385e60bc2adb6d1e`
Instagram Profile Scraper	`5c55ffbf10fbbc7ee63ebb2bcaa332b7`
Instagram Reels Scraper	`4aad3ef0de6a5cea65b48dd575531a8c`
LaCentrale Cars Scraper	`665c260fd9139121a4530d23b35b26db`
LaCentrale Listing Status Checker	`1d7fbb550feb1cf93e3e439039bcd687`
Leboncoin Auto Message Sender	`fa9c988768a3d61f358916400f3c4b65`
Leboncoin Boutiques Scraper	`c6e88128aef71c079e58f0518687e10c`
Leboncoin Listing Scraper	`9eade2d2a693bd871851806650e7fb4e`
Leboncoin Listing Status Checker	`0c60c33b95db65c86d5c9fc127b7b2aa`
Leboncoin Listings & Phone Search Export	`7bc4acdb18f2b90fdd5eb42b8e8251e9`
Leboncoin Listings Search Export	`33db1ca85160105eeb84d5aa51cfad10`
LinkedIn Leads Scraper	`aa164b3c44d4f435fe5e04ddf6757296`
LinkedIn Posts Commenters and Likers Scraper	`8a309e0e0df48511111a9da6382b2ef8`
LinkedIn Posts Scraper	`6a9159b682cfb187977ae5c645e4df7a`
LinkedIn Profile & Email Scraper (No Login)	`e7c4200bab5942e5329f711e26c0666b`
LinkedIn Profile Scraper	`5c11752d8687df2332c08247c4fb655a`
Pages Jaunes Companies Search Export	`cd3e6b39d78365cd9d0ffbe1c7f5e5bd`
PAP Search Export	`9980d62d6a23161a08d1134c0cabdf5c`
ParuVendu Listings Scraper	`6a564030e814e6de4931637f7fb0c06e`
Realtor Property Listings Scraper	`f388a9866f943bc5c29bde8b33587bd3`
Reddit Scraper	`6f99262f742b533d96936bf4088fba3e`
Sales Navigator Companies Scraper	`6b66cba236b64604581864230e1f4d36`
Sales Navigator Leads Scraper	`a1d243d3824a1ac773414416eb80362d`
Sales Navigator Profile Scraper	`40d08f9bfec4a8775bb5586ce7b33a35`
SeLoger Search Export	`78f5839ee4b97c30e67eec391b907dd0`
TikTok Comments Scraper	`5fc7b468267ec881344a11ac05c15a76`
TikTok Top Ads Scraper	`b5079ede3002655d286a03ad29604ac6`
TikTok Video Scraper	`1e6cba50f2713539a2b0ac784184ed92`
TripAdvisor Restaurants Search Export	`f781435f026b36b19ef74d591a077cb7`
TripAdvisor Reviews Scraper	`83970312f2402accbb3553b326a2c8eb`
Trustpilot Reviews Scraper	`b3362ab52c6fab3d8897af79cbba380e`
Twitter Followers & Following Scraper	`8c6eed231ba0378c42efbe391f70d6d8`
Twitter Profile Scraper	`6c5227261fbbd3430f926d409f4c44bf`
Twitter Search Results Scraper	`1b16ff414d27920fb325b68436dbf5fc`
Twitter User Tweets Scraper	`9d6b83aaf1ae9d5b4a775d223e3eb5df`
Vinted Products Scraper	`ffd34f9b42a79b7323a048f09fc158e6`
Yelp Reviews Scraper	`ea1c83f60e32a4e1de91a1ed1e41b268`
Yelp Search Export	`ac92974d3c1c9f8d3997767f090fa0b8`
YouTube Channel Scraper	`8dce74bb8c2330fdafee70baa22b4709`
YouTube Comments Scraper	`66e9fbe07ae0a543576dc6ff241c229c`

Code Examples

curl -X GET "https://api.lobstr.io/v1/crawlers" \
  -H "Authorization: Token YOUR_API_KEY"

import requests

url = "https://api.lobstr.io/v1/crawlers"
headers = {
    "Authorization": "Token YOUR_API_KEY"
}

response = requests.get(url, headers=headers)
data = response.json()

print(f"Total crawlers: {data['total_results']}")
print(f"Showing {data['result_from']}-{data['result_to']}\n")

# Display available crawlers
for crawler in data['data']:
    status = "✓" if crawler['is_available'] else "✗"
    premium = " [PREMIUM]" if crawler['is_premium'] else ""
    print(f"{status} {crawler['name']}{premium}")
    print(f"   ID: {crawler['id']}")
    print(f"   Credits: {crawler['credits_per_row']} per row")
    print()

Response

200

{
  "total_results": 56,
  "limit": 50,
  "page": 1,
  "total_pages": 2,
  "result_from": 1,
  "result_to": 50,
  "data": [
    {
      "id": "6c5227261fbbd3430f926d409f4c44bf",
      "object": "crawler",
      "account": null,
      "name": "Twitter Profile Scraper",
      "slug": "twitter-profile-scraper",
      "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+",
      "is_public": true,
      "is_available": true,
      "is_premium": false,
      "has_issues": false,
      "max_concurrency": 20,
      "credits_per_row": 2
    },
    {
      "id": "c47dfc2beba9fdc2472d3b3321b06b66",
      "object": "crawler",
      "account": {
        "type": "facebook-sync",
        "baseurl": "https://www.facebook.com",
        "cookies": [
          {
            "name": "c_user",
            "required": true
          },
          {
            "name": "xs",
            "required": true
          }
        ],
        "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+"
      },
      "name": "Facebook Pages Search Export",
      "slug": "facebook-pages-search-export",
      "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+",
      "is_public": true,
      "is_available": true,
      "is_premium": false,
      "has_issues": false,
      "max_concurrency": 20,
      "credits_per_row": 20
    },
    {
      "id": "a1d243d3824a1ac773414416eb80362d",
      "object": "crawler",
      "account": {
        "type": "sales-nav-sync",
        "baseurl": "https://www.linkedin.com",
        "cookies": [
          {
            "name": "li_at",
            "required": true
          },
          {
            "name": "JSESSIONID",
            "required": true
          }
        ],
        "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+"
      },
      "name": "Sales Navigator Leads Scraper",
      "slug": "sales-navigator-leads-scraper",
      "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+",
      "is_public": true,
      "is_available": true,
      "is_premium": false,
      "has_issues": false,
      "max_concurrency": 10,
      "credits_per_row": 3
    }
  ],
  "next": null,
  "previous": null
}

Overview

Getting Started

Core Concepts

User

Crawler

Squid

Delivery

Account

Task

Run

Result

List Crawlers

Headers

Response Structure

Crawler Object Fields

Available Crawlers

Code Examples

Response

​Headers

​Response Structure

​Crawler Object Fields

​Available Crawlers

​Code Examples

​Response

Headers

Response Structure

Crawler Object Fields

Available Crawlers

Code Examples

Response