List Squids

curl --request GET \
  --url https://api.lobstr.io/v1/squids \
  --header 'Authorization: <authorization>'

import requests

url = "https://api.lobstr.io/v1/squids"

headers = {"Authorization": "<authorization>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/squids', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/squids",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/squids"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/squids")
  .header("Authorization", "<authorization>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/squids")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "total_results": 123,
  "limit": 123,
  "page": 123,
  "total_pages": 123,
  "result_from": 123,
  "result_to": 123,
  "data": [
    {}
  ],
  "next": {},
  "previous": {},
  "data[].id": "<string>",
  "data[].name": "<string>",
  "data[].created_at": "<string>",
  "data[].is_active": true,
  "data[].is_ready": true,
  "data[].crawler": "<string>",
  "data[].crawler_name": "<string>",
  "data[].icon": "<string>",
  "data[].is_public": true,
  "data[].concurrency": 123,
  "data[].params": {},
  "data[].to_complete": true,
  "data[].export_unique_results": true,
  "data[].no_line_breaks": true,
  "data[].last_run_status": "<string>",
  "data[].last_run_at": "<string>",
  "data[].total_runs": 123,
  "data[].next_launch_at": {},
  "data[].google_sheet_fields": {},
  "data[].ftp_fields": {},
  "data[].s3_fields": {},
  "data[].webhook_fields": {},
  "data[].schedule": {},
  "data[].schedule_time": "<string>",
  "data[].download_url": {}
}

GET

squids

List Squids

curl --request GET \
  --url https://api.lobstr.io/v1/squids \
  --header 'Authorization: <authorization>'

import requests

url = "https://api.lobstr.io/v1/squids"

headers = {"Authorization": "<authorization>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: '<authorization>'}};

fetch('https://api.lobstr.io/v1/squids', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/squids",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/squids"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "<authorization>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.lobstr.io/v1/squids")
  .header("Authorization", "<authorization>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/squids")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = '<authorization>'

response = http.request(request)
puts response.read_body

{
  "total_results": 123,
  "limit": 123,
  "page": 123,
  "total_pages": 123,
  "result_from": 123,
  "result_to": 123,
  "data": [
    {}
  ],
  "next": {},
  "previous": {},
  "data[].id": "<string>",
  "data[].name": "<string>",
  "data[].created_at": "<string>",
  "data[].is_active": true,
  "data[].is_ready": true,
  "data[].crawler": "<string>",
  "data[].crawler_name": "<string>",
  "data[].icon": "<string>",
  "data[].is_public": true,
  "data[].concurrency": 123,
  "data[].params": {},
  "data[].to_complete": true,
  "data[].export_unique_results": true,
  "data[].no_line_breaks": true,
  "data[].last_run_status": "<string>",
  "data[].last_run_at": "<string>",
  "data[].total_runs": 123,
  "data[].next_launch_at": {},
  "data[].google_sheet_fields": {},
  "data[].ftp_fields": {},
  "data[].s3_fields": {},
  "data[].webhook_fields": {},
  "data[].schedule": {},
  "data[].schedule_time": "<string>",
  "data[].download_url": {}
}

This endpoint returns a paginated list of all squids belonging to the authenticated user. You can search by name and control pagination with query parameters.

Headers

string

required

Your API authentication token. Value: Token YOUR_API_KEY

Query Parameters

string

Filter squids by name (case-insensitive partial match). Example: Google

integer

Number of results per page. Example: 10

integer

Page number for pagination. Example: 2

Response Structure

integer

Total number of squids matching the query. Example: 15

integer

Number of results per page. Example: 50

integer

Current page number. Example: 1

integer

Total number of pages available. Example: 1

integer

Starting index of results in current page. Example: 1

integer

Ending index of results in current page. Example: 15

array

Array of squid objects with comprehensive details

string | null

Full URL to the next page (null if on last page). Example: null

string | null

Full URL to the previous page (null if on first page). Example: null

Squid Object Fields

string

Unique squid identifier. Example: "c106a44a98044ef18acc59986ae10967"

string

Display name of the squid. Example: "Google Maps (1)"

string

Timestamp when the squid was created. Example: "2024-01-15T10:30:00Z"

boolean

Whether the squid is active and can run. Example: true

boolean

Whether the squid has tasks and is ready to run. Example: true

string

Unique identifier of the associated crawler. Example: "4734d096159ef05210e0e1677e8be823"

string

Display name of the crawler. Example: "Google Maps Search Export"

string

URL to the crawler’s icon image. Example: "https://cdn.lobstr.io/icons/google-maps.png"

boolean

Whether the crawler is publicly available. Example: true

integer

Number of concurrent tasks the squid runs. Example: 2

object

Crawler-specific parameter configuration object. Example: {"language": "en"}

boolean

Whether to stop after completing all tasks. Example: false

boolean

Whether to export only unique results. Example: true

boolean

Whether to remove line breaks from exported data. Example: false

string

Status of the most recent run (completed, running, error, etc.). Example: "DONE"

string

Timestamp of the most recent run. Example: "2024-01-20T15:45:00Z"

integer

Total number of times the squid has been executed. Example: 5

string | null

Scheduled time for the next automated run (null if not scheduled). Example: "2024-01-21T10:00:00Z"

object | null

Google Sheets integration configuration (null if not configured). Example: null

object | null

FTP integration configuration (null if not configured). Example: null

object | null

AWS S3 integration configuration (null if not configured). Example: null

object | null

Webhook integration configuration (null if not configured). Example: null

string | null

Cron expression for scheduled runs (null if not scheduled). Example: "0 10 * * *"

string

Human-readable schedule description. Example: "Daily at 10:00 AM"

string | null

Temporary signed URL to download results (null if no results available). Example: "https://storage.lobstr.io/downloads/xyz123..."

Use the ‘name’ query parameter to quickly find squids. It performs a case-insensitive partial match on squid names.

The ‘next’ and ‘previous’ fields contain full URLs for easy pagination. Use them directly to navigate through pages.

Check ‘last_run_status’ to see if a squid has completed, is running, or encountered errors. Useful for monitoring multiple squids.

Squids with ‘is_ready: false’ haven’t been configured with tasks yet. Add tasks before starting a run.

The ‘download_url’ field provides a temporary signed URL to download results. URLs expire after a certain time period.

Code Examples

curl -X GET "https://api.lobstr.io/v1/squids?name=Google&limit=10&page=1" \
  -H "Authorization: Token YOUR_API_KEY"

import requests

url = "https://api.lobstr.io/v1/squids"
headers = {
    "Authorization": "Token YOUR_API_KEY"
}
params = {
    "name": "Google",
    "limit": 10,
    "page": 1
}

response = requests.get(url, headers=headers, params=params)
data = response.json()

print(f"Total squids found: {data['total_results']}")
print(f"Page {data['page']} of {data['total_pages']}\n")

for squid in data['data']:
    status = "Active" if squid['is_active'] else "Inactive"
    print(f"{squid['name']} ({status})")
    print(f"  ID: {squid['id']}")
    print(f"  Crawler: {squid['crawler_name']}")
    print(f"  Total Runs: {squid['total_runs']}")
    print(f"  Last Status: {squid['last_run_status'] or 'Never run'}\n")

if data['next']:
    print(f"Next page: {data['next']}")

Response

200

{
  "total_results": 34,
  "limit": 10,
  "page": 1,
  "total_pages": 4,
  "result_from": 1,
  "result_to": 10,
  "data": [
    {
      "id": "e86b29c032024b66aff529e1d43c2bd7",
      "created_at": "2025-02-03T14:24:23Z",
      "name": "My Google Maps Scraper",
      "is_ready": true,
      "is_active": true,
      "concurrency": 2,
      "params": {
        "country": "United States",
        "ratings": "4.0+",
        "language": "English (United States)",
        "max_results": 150
      },
      "crawler": "4734d096159ef05210e0e1677e8be823",
      "crawler_name": "Google Maps Search Export",
      "last_run_status": "DONE",
      "total_runs": 5,
      "last_run_at": "2025-03-05T09:27:17.012423",
      "next_launch_at": null,
      "is_public": true
    }
  ],
  "next": "https://api.lobstr.io/v1/squids?limit=10&page=2",
  "previous": null
}

Update Squid Get Squid Details

Overview

Getting Started

Core Concepts

User

Crawler

Squid

Delivery

Account

Task

Run

Result

List Squids

Headers

Query Parameters

Response Structure

Squid Object Fields

Code Examples

Response

​Headers

​Query Parameters

​Response Structure

​Squid Object Fields

​Code Examples

​Response

Headers

Query Parameters

Response Structure

Squid Object Fields

Code Examples

Response