Get Crawler Parameters
curl --request GET \
--url https://api.lobstr.io/v1/crawlers/{crawler_hash}/params \
--header 'Authorization: <authorization>'import requests
url = "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params"
headers = {"Authorization": "<authorization>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: '<authorization>'}};
fetch('https://api.lobstr.io/v1/crawlers/{crawler_hash}/params', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "<authorization>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/crawlers/{crawler_hash}/params")
.header("Authorization", "<authorization>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/crawlers/{crawler_hash}/params")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'
response = http.request(request)
puts response.read_body{
"task.{parameter_name}.type": "<string>",
"task.{parameter_name}.group": "<string>",
"task.{parameter_name}.regex": "<string>",
"task.{parameter_name}.required": true,
"task.{parameter_name}.default": {},
"task.{parameter_name}.attribute": "<string>",
"task.{parameter_name}.is_params": true,
"squid.{parameter_name}.type": "<string>",
"squid.{parameter_name}.allowed": [
{}
],
"squid.{parameter_name}.default": {},
"squid.{parameter_name}.required": true,
"squid.{parameter_name}.attribute": "<string>",
"squid.functions.{function_name}.sort": 123,
"squid.functions.{function_name}.default": true,
"squid.functions.{function_name}.credits_per_function": 123
}Crawler
Get Crawler Parameters
Retrieve the list of configurable input parameters for a specific crawler
GET
/
v1
/
crawlers
/
{crawler_hash}
/
params
Get Crawler Parameters
curl --request GET \
--url https://api.lobstr.io/v1/crawlers/{crawler_hash}/params \
--header 'Authorization: <authorization>'import requests
url = "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params"
headers = {"Authorization": "<authorization>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: '<authorization>'}};
fetch('https://api.lobstr.io/v1/crawlers/{crawler_hash}/params', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/crawlers/{crawler_hash}/params"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "<authorization>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/crawlers/{crawler_hash}/params")
.header("Authorization", "<authorization>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/crawlers/{crawler_hash}/params")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'
response = http.request(request)
puts response.read_body{
"task.{parameter_name}.type": "<string>",
"task.{parameter_name}.group": "<string>",
"task.{parameter_name}.regex": "<string>",
"task.{parameter_name}.required": true,
"task.{parameter_name}.default": {},
"task.{parameter_name}.attribute": "<string>",
"task.{parameter_name}.is_params": true,
"squid.{parameter_name}.type": "<string>",
"squid.{parameter_name}.allowed": [
{}
],
"squid.{parameter_name}.default": {},
"squid.{parameter_name}.required": true,
"squid.{parameter_name}.attribute": "<string>",
"squid.functions.{function_name}.sort": 123,
"squid.functions.{function_name}.default": true,
"squid.functions.{function_name}.credits_per_function": 123
}This endpoint returns all configurable input parameters for a specific crawler. The response is organized into two main sections:
task and squid.
Response Structure
The response contains:- task: Object containing task-level parameters (parameter name as key)
- squid: Object containing squid-level parameters and functions
- squid.functions: Optional add-on features with extra credit costs
Task-level Parameter Fields
string
Data type (string, int, boolean, etc.). Example:
"string"string
Logical grouping (e.g., “url”, “location”). Example:
"url"string
Validation pattern for string inputs. Example:
"^https?://.*"boolean
Whether the parameter is mandatory. Example:
truestring | number | boolean | null
Default value if not provided. Example:
nullstring
Internal attribute name. Example:
"url"boolean
Whether it’s used for URL parameter generation. Example:
trueSquid-level Parameter Fields
string
Data type of the parameter. Example:
"string"array
Array of valid values (for enum types). Example:
["en", "es", "fr"]string | number | boolean | null
Default value if not provided. Example:
"en"boolean
Whether the parameter is mandatory. Example:
falsestring
Internal attribute name. Example:
"language"Functions (Optional Add-ons) Fields
integer
Display order of the function. Example:
1boolean
Whether the function is enabled by default. Example:
falsenumber
Extra credits charged per row when this function is enabled. Example:
0.5Headers
string
required
Your API authentication token. Value:
Token YOUR_API_KEYstring
default:"application/json"
Content type of the request. Value:
application/jsonPath Parameters
string
required
The unique identifier (hash/id) of the crawler. Example:
4734d096159ef05210e0e1677e8be823The response structure uses parameter names as object keys. Access task parameters via params.task.url, squid parameters via params.squid.language, etc.
Functions under squid.functions are optional add-ons. Each function has a credits_per_function cost applied per result row when enabled.
Pay attention to ‘allowed’ arrays in squid parameters - these only accept specific values. Sending invalid values will cause validation errors.
Use ‘regex’ patterns in task parameters to validate input before sending. This helps catch errors early and reduces failed tasks.
Parameters with ‘group’ field are logically related. For location-based crawlers, you’ll typically need to provide all parameters in the ‘location’ group.
Code Examples
curl -X GET "https://api.lobstr.io/v1/crawlers/4734d096159ef05210e0e1677e8be823/params" \
-H "Authorization: Token YOUR_API_KEY"
import requests
crawler_id = "4734d096159ef05210e0e1677e8be823"
url = f"https://api.lobstr.io/v1/crawlers/{crawler_id}/params"
headers = {
"Authorization": "Token YOUR_API_KEY"
}
response = requests.get(url, headers=headers)
params = response.json()
# Display task-level parameters
print("=== Task-level Parameters ===")
for param_name, param_config in params['task'].items():
req = " (Required)" if param_config['required'] else " (Optional)"
print(f"\n{param_name} ({param_config['type']}){req}")
if param_config.get('regex'):
print(f" Validation: {param_config['regex']}")
if param_config.get('default') is not None:
print(f" Default: {param_config['default']}")
print(f" Group: {param_config.get('group', 'N/A')}")
# Display squid-level parameters
print("\n=== Squid-level Parameters ===")
squid_params = {k: v for k, v in params['squid'].items() if k != 'functions'}
for param_name, param_config in squid_params.items():
req = " (Required)" if param_config['required'] else " (Optional)"
print(f"\n{param_name} ({param_config['type']}){req}")
if param_config.get('allowed'):
print(f" Allowed values: {len(param_config['allowed'])} options")
print(f" Default: {param_config['default']}")
# Display optional functions (add-ons with extra costs)
if 'functions' in params['squid']:
print("\n=== Optional Functions (Extra Credits) ===")
for func_name, func_config in params['squid']['functions'].items():
enabled = " (Enabled by default)" if func_config['default'] else ""
print(f"\n{func_name}{enabled}")
print(f" Credits per function: {func_config['credits_per_function']}")
print(f" Sort order: {func_config['sort']}")
Response
200
{
"task": {
"url": {
"type": "string",
"group": "url",
"regex": "http(.*)google(.*)/maps/(search|place)(.*)",
"default": null,
"required": true,
"attribute": "url",
"is_params": false
},
"city": {
"type": "string",
"group": "location",
"default": null,
"required": true,
"attribute": "city",
"is_params": true
},
"region": {
"type": "string",
"group": "location",
"default": null,
"required": false,
"attribute": "region",
"is_params": true
},
"country": {
"type": "string",
"group": "location",
"default": null,
"required": true,
"attribute": "country",
"is_params": true
},
"category": {
"type": "string",
"group": "location",
"default": null,
"required": true,
"attribute": "category",
"is_params": true
},
"district": {
"type": "string",
"group": "location",
"default": null,
"required": false,
"attribute": "district",
"is_params": true
}
},
"squid": {
"country": {
"type": "string",
"allowed": [
"Afghanistan",
"Albania",
"Algeria",
"United States",
"United Kingdom",
"France",
"Germany",
"Japan",
"Brazil"
],
"default": "United States",
"required": false,
"attribute": "country",
"is_params": false
},
"ratings": {
"type": "string",
"allowed": [
"Any rating",
"2.0+",
"2.5+",
"3.0+",
"3.5+",
"4.0+",
"4.5+"
],
"default": "Any rating",
"required": false,
"attribute": "ratings",
"is_params": false
},
"language": {
"type": "string",
"allowed": [
"Afrikaans",
"English (United States)",
"Español (España)",
"Français (France)",
"Deutsch (Deutschland)",
"Italiano",
"Português (Brasil)",
"日本語",
"简体中文",
"繁體中文"
],
"default": "English (United States)",
"required": true,
"attribute": "language",
"is_params": false
},
"functions": {
"fetch_business_images": {
"sort": 1,
"default": false,
"credits_per_function": 10
},
"collect_business_details": {
"sort": 0,
"default": false,
"credits_per_function": 10
},
"extract_emails_from_website": {
"sort": 2,
"default": true,
"credits_per_function": 10
}
}
}
}