Create Squid
curl --request POST \
--url https://api.lobstr.io/v1/squids \
--header 'Authorization: <authorization>' \
--header 'Content-Type: <content-type>' \
--data '
{
"crawler": "<string>",
"name": "<string>"
}
'import requests
url = "https://api.lobstr.io/v1/squids"
payload = {
"crawler": "<string>",
"name": "<string>"
}
headers = {
"Authorization": "<authorization>",
"Content-Type": "<content-type>"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
body: JSON.stringify({crawler: '<string>', name: '<string>'})
};
fetch('https://api.lobstr.io/v1/squids', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/squids",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'crawler' => '<string>',
'name' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>",
"Content-Type: <content-type>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/squids"
payload := strings.NewReader("{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<authorization>")
req.Header.Add("Content-Type", "<content-type>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/squids")
.header("Authorization", "<authorization>")
.header("Content-Type", "<content-type>")
.body("{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/squids")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"crawler": "<string>",
"is_active": true,
"concurrency": 123,
"params": {},
"schedule": {},
"to_complete": true
}Squid
Create Squid
Create a new squid container for a specific crawler to organize and run scraping tasks
POST
/
v1
/
squids
Create Squid
curl --request POST \
--url https://api.lobstr.io/v1/squids \
--header 'Authorization: <authorization>' \
--header 'Content-Type: <content-type>' \
--data '
{
"crawler": "<string>",
"name": "<string>"
}
'import requests
url = "https://api.lobstr.io/v1/squids"
payload = {
"crawler": "<string>",
"name": "<string>"
}
headers = {
"Authorization": "<authorization>",
"Content-Type": "<content-type>"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
body: JSON.stringify({crawler: '<string>', name: '<string>'})
};
fetch('https://api.lobstr.io/v1/squids', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/squids",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'crawler' => '<string>',
'name' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>",
"Content-Type: <content-type>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/squids"
payload := strings.NewReader("{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<authorization>")
req.Header.Add("Content-Type", "<content-type>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/squids")
.header("Authorization", "<authorization>")
.header("Content-Type", "<content-type>")
.body("{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/squids")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n \"crawler\": \"<string>\",\n \"name\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"crawler": "<string>",
"is_active": true,
"concurrency": 123,
"params": {},
"schedule": {},
"to_complete": true
}This endpoint creates a new squid for a specified crawler by providing the crawler’s hash ID. A squid is a container that groups together related tasks and configurations for a specific scraping operation.
What is a Squid?
A squid acts as a project workspace for your scraping tasks:- Groups tasks: All URLs or items you want to scrape with the same crawler
- Stores configuration: Crawler parameters, concurrency settings, delivery options
- Manages runs: Tracks execution history and results
- Enables scheduling: Set up automated recurring scrapes
Headers
string
required
Your API authentication token. Value:
Token YOUR_API_KEYstring
required
Must be application/json. Value:
application/jsonRequest Body
string
required
The unique ID (hash) of the crawler to use for this squid. Example:
"4734d096159ef05210e0e1677e8be823"string
Custom name for the squid. If not provided, an auto-generated name will be used. Example:
"My Google Maps Scraper"Response Field Explanations
string
Unique squid identifier. Example:
"c106a44a98044ef18acc59986ae10967"string
Squid name (auto-generated as “Crawler Name (N)” if not provided). Example:
"Google Maps (1)"string
ID of the associated crawler. Example:
"4734d096159ef05210e0e1677e8be823"boolean
Whether the squid is active and can run. Example:
trueinteger
Number of concurrent tasks (default: 1). Example:
1object
Squid-level parameters with default values from crawler. Example:
{}object | null
Cron schedule configuration (null if not scheduled). Example:
nullboolean
Whether to stop after all tasks complete (false = run continuously). Example:
falseAfter creating a squid, you can update its parameters and settings using the Update Squid endpoint before adding tasks.
The squid automatically inherits default parameters from the crawler. Check the response ‘params’ field to see what defaults were applied.
Custom names help organize multiple squids using the same crawler. Without a custom name, squids are numbered sequentially.
Make sure to get the crawler ID from the List Crawlers endpoint first. Using an invalid crawler ID will result in an error.
Code Examples
curl -X POST "https://api.lobstr.io/v1/squids" \
-H "Authorization: Token YOUR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"crawler": "4734d096159ef05210e0e1677e8be823",
"name": "My Google Maps Scraper"
}'
import requests
url = "https://api.lobstr.io/v1/squids"
headers = {
"Authorization": "Token YOUR_API_KEY",
"Content-Type": "application/json"
}
payload = {
"crawler": "4734d096159ef05210e0e1677e8be823",
"name": "My Google Maps Scraper"
}
response = requests.post(url, headers=headers, json=payload)
squid = response.json()
print(f"Squid created successfully!")
print(f"ID: {squid['id']}")
print(f"Name: {squid['name']}")
print(f"Crawler: {squid['crawler']}")
print(f"Is Active: {squid['is_active']}")
print(f"Concurrency: {squid['concurrency']}")
print(f"\nDefault Parameters:")
for key, value in squid['params'].items():
print(f" {key}: {value}")
Response
201
{
"id": "e86b29c032024b66aff529e1d43c2bd7",
"account": [],
"concurrency": 1,
"crawler": "4734d096159ef05210e0e1677e8be823",
"created_at": "2025-02-03T14:24:23Z",
"is_active": true,
"name": "My Google Maps Scraper",
"params": {
"max_results": 200,
"ratings": "Any rating",
"country": "United States",
"language": "English (United States)"
},
"schedule": null,
"to_complete": false
}