Get Crawler Details

curl --request GET \
  --url https://api.lobstr.io/v1/crawlers/{crawler_hash} \
  --header 'Authorization: <authorization>'

import requests

url = "https://api.lobstr.io/v1/crawlers/{crawler_hash}"

headers = {"Authorization": "<authorization>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/crawlers/{crawler_hash}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/crawlers/{crawler_hash}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/crawlers/{crawler_hash}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/crawlers/{crawler_hash}")
  .header("Authorization", "<authorization>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/crawlers/{crawler_hash}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "name": "<string>",
  "slug": "<string>",
  "is_public": true,
  "is_available": true,
  "is_premium": true,
  "has_issues": true,
  "account": {},
  "input": [
    {}
  ],
  "result": [
    {}
  ],
  "input[].level": "<string>",
  "input[].is_params": "<string>",
  "input[].is_details": "<string>",
  "input[].function": "<string>",
  "input[].credits_per_function": "<string>",
  "input[].name": "<string>",
  "input[].type": "<string>",
  "input[].required": "<string>",
  "input[].default": "<string>"
}

GET

crawlers

{crawler_hash}

Get Crawler Details

curl --request GET \
  --url https://api.lobstr.io/v1/crawlers/{crawler_hash} \
  --header 'Authorization: <authorization>'

import requests

url = "https://api.lobstr.io/v1/crawlers/{crawler_hash}"

headers = {"Authorization": "<authorization>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/crawlers/{crawler_hash}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/crawlers/{crawler_hash}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/crawlers/{crawler_hash}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/crawlers/{crawler_hash}")
  .header("Authorization", "<authorization>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/crawlers/{crawler_hash}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "name": "<string>",
  "slug": "<string>",
  "is_public": true,
  "is_available": true,
  "is_premium": true,
  "has_issues": true,
  "account": {},
  "input": [
    {}
  ],
  "result": [
    {}
  ],
  "input[].level": "<string>",
  "input[].is_params": "<string>",
  "input[].is_details": "<string>",
  "input[].function": "<string>",
  "input[].credits_per_function": "<string>",
  "input[].name": "<string>",
  "input[].type": "<string>",
  "input[].required": "<string>",
  "input[].default": "<string>"
}

This endpoint retrieves detailed information about a specific crawler using its hash ID. It helps you understand each crawler’s availability, cost, parameters, and potential issues before using it.

Headers

string

required

Your API authentication token. Value: Token YOUR_API_KEY

string

default:"application/json"

Content type of the request. Value: application/json

Query Parameters

string

required

The unique identifier (hash/id) of the crawler. Example: 4734d096159ef05210e0e1677e8be823

Response Field Explanations

string

Unique identifier for the crawler. Example: "c106a44a98044ef18acc59986ae10967"

string

Human-readable name (e.g., “Google Maps Search Export”). Example: "Google Maps (1)"

string

URL-friendly identifier. Example: ""

boolean

Whether the crawler is publicly accessible. Example: true

boolean

Whether the crawler is available for runs (false if paused). Example: true

boolean

Requires premium subscription or extra credits. Example: false

boolean

Currently experiencing issues (runs paused until fixed). Example: false

object | null

Required account sync details (null if no authentication needed). Example: null

array

Array of configurable parameters (task-level and squid-level). Example: []

array

Array of output field names this crawler collects. Example: []

Understanding Input Parameters

string

Parameter level: “task” (per URL/task) or “squid” (overall run behavior). Example: ""

string

For task-level: true indicates this parameter is used per task submission. Example: ""

string

For squid-level: true indicates this controls overall behavior or features. Example: ""

string

If true, this is an optional add-on feature with extra credit costs. Example: ""

string

Extra credits charged per row when this optional function is enabled. Example: ""

string

Parameter name to use when configuring the crawler. Example: "Google Maps (1)"

string

Data type of the parameter (string, int, boolean, etc.). Example: ""

string

Whether this parameter is mandatory. Example: ""

string

Default value if parameter is not provided. Example: ""

Check the ‘has_issues’ flag before creating tasks. If true, the crawler is experiencing problems and runs are paused.

Parameters with ‘function: true’ are optional add-ons that cost extra credits per row (specified in ‘credits_per_function’).

Some crawlers require account synchronization (see ‘account’ field). You’ll need to sync cookies before running these crawlers.

Use the ‘result’ array to know exactly what fields will be returned in your data. This helps you plan your data processing pipeline.

Code Examples

curl -X GET "https://api.lobstr.io/v1/crawlers/4734d096159ef05210e0e1677e8be823" \
  -H "Authorization: Token YOUR_API_KEY"

import requests

crawler_id = "4734d096159ef05210e0e1677e8be823"
url = f"https://api.lobstr.io/v1/crawlers/{crawler_id}"
headers = {
    "Authorization": "Token YOUR_API_KEY"
}

response = requests.get(url, headers=headers)
crawler = response.json()

print(f"Crawler: {crawler['name']}")
print(f"Status: {'Available' if crawler['is_available'] else 'Unavailable'}")
print(f"Has Issues: {crawler['has_issues']}")
print(f"Credits per row: {crawler['credits_per_row']}\n")

# Display task-level inputs
print("Task-level parameters:")
task_inputs = [i for i in crawler['input'] if i['level'] == 'task']
for param in task_inputs:
    req = "Required" if param['required'] else "Optional"
    print(f"  - {param['name']} ({param['type']}) - {req}")

# Display squid-level inputs with functions
print("\nSquid-level parameters:")
squid_inputs = [i for i in crawler['input'] if i['level'] == 'squid']
for param in squid_inputs:
    cost = f" (+{param['credits_per_function']} credits)" if param.get('function') else ""
    print(f"  - {param['name']}{cost}")

# Display output fields
print(f"\nOutput fields ({len(crawler['result'])} total):")
print(", ".join(crawler['result'][:10]) + "...")

Response

200

{
  "id": "4734d096159ef05210e0e1677e8be823",
  "object": "crawler",
  "account": null,
  "name": "Google Maps Search Export",
  "slug": "google-maps-search-export",
  "icon": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MCIgaGVpZ2h0PSI1MCI+PC9zdmc+",
  "is_public": true,
  "is_available": true,
  "is_premium": false,
  "has_issues": false,
  "input": [
    {
      "name": "url",
      "type": "string",
      "level": "task",
      "regex": "http(.*)google(.*)/maps/(search|place)(.*)",
      "default": null,
      "example": "https://www.google.com/maps/search/restaurant/@43.2928346,5.3662584,14z",
      "required": false,
      "is_details": false,
      "description": "A string representing the Google Maps Search URL, containing all establishments the crawler will collect."
    },
    {
      "name": "activity",
      "type": "string",
      "level": "task",
      "value": "restaurant",
      "example": "restaurant",
      "required": false,
      "is_params": true,
      "description": "The activity of the search location."
    },
    {
      "max": 200,
      "name": "max_results",
      "type": "int",
      "level": "squid",
      "default": 200,
      "example": 200,
      "required": false,
      "is_details": true,
      "description": "Google Maps restricts search results to a maximum of 200. To access a larger dataset, consider utilizing our URL generator."
    },
    {
      "name": "extract_emails_from_website",
      "type": "boolean",
      "level": "squid",
      "default": true,
      "display": "Extract Emails From Website",
      "function": true,
      "priority": 3,
      "required": false,
      "is_details": true,
      "description": "Extract email addresses from the business's website.",
      "credits_per_function": 10
    },
    {
      "name": "collect_business_details",
      "type": "boolean",
      "level": "squid",
      "default": false,
      "display": "Collect Business Details",
      "function": true,
      "priority": 1,
      "required": false,
      "is_details": true,
      "description": "Extract attributes like plus_code, poi, health, and last_opening_hours_updated_at.",
      "credits_per_function": 10
    }
  ],
  "result": [
    "zero_x",
    "cid",
    "url",
    "name",
    "address",
    "country_name",
    "country_code",
    "zip_code",
    "city",
    "lat",
    "lng",
    "ratings",
    "score",
    "category",
    "main_image_url",
    "phone",
    "website",
    "menu",
    "plus_code",
    "is_temporarily_closed",
    "is_permanently_closed",
    "has_owner",
    "opening_hours",
    "email",
    "facebook",
    "instagram"
  ],
  "max_concurrency": 20,
  "credits_per_row": 2
}

List Crawlers Get Crawler Parameters

Overview

Getting Started

Core Concepts

User

Crawler

Squid

Delivery

Account

Task

Run

Result

Get Crawler Details

Headers

Query Parameters

Response Field Explanations

Understanding Input Parameters

Code Examples

Response

​Headers

​Query Parameters

​Response Field Explanations

​Understanding Input Parameters

​Code Examples

​Response

Headers

Query Parameters

Response Field Explanations

Understanding Input Parameters

Code Examples

Response