Configure Webhook Delivery

curl --request POST \
  --url https://api.lobstr.io/v1/delivery \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: <content-type>' \
  --data '
{
  "webhook_fields.url": "<string>",
  "webhook_fields.is_active": true,
  "webhook_fields.retry": true,
  "webhook_fields.events.run.running": true,
  "webhook_fields.events.run.paused": true,
  "webhook_fields.events.run.done": true,
  "webhook_fields.events.run.error": true
}
'

import requests

url = "https://api.lobstr.io/v1/delivery"

payload = {
    "webhook_fields.url": "<string>",
    "webhook_fields.is_active": True,
    "webhook_fields.retry": True,
    "webhook_fields.events.run.running": True,
    "webhook_fields.events.run.paused": True,
    "webhook_fields.events.run.done": True,
    "webhook_fields.events.run.error": True
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "<content-type>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
  body: JSON.stringify({
    'webhook_fields.url': '<string>',
    'webhook_fields.is_active': true,
    'webhook_fields.retry': true,
    'webhook_fields.events.run.running': true,
    'webhook_fields.events.run.paused': true,
    'webhook_fields.events.run.done': true,
    'webhook_fields.events.run.error': true
  })
};

fetch('https://api.lobstr.io/v1/delivery', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/delivery",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'webhook_fields.url' => '<string>',
    'webhook_fields.is_active' => true,
    'webhook_fields.retry' => true,
    'webhook_fields.events.run.running' => true,
    'webhook_fields.events.run.paused' => true,
    'webhook_fields.events.run.done' => true,
    'webhook_fields.events.run.error' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: <content-type>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/delivery"

	payload := strings.NewReader("{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "<content-type>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/delivery")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "<content-type>")
  .body("{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/delivery")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}"

response = http.request(request)
puts response.read_body

{
  "webhook_fields.url": "<string>",
  "webhook_fields.is_active": true,
  "webhook_fields.retry": true,
  "webhook_fields.events.run.running": true,
  "webhook_fields.events.run.paused": true,
  "webhook_fields.events.run.done": true,
  "webhook_fields.events.run.error": true
}

POST

delivery

Configure Webhook Delivery

curl --request POST \
  --url https://api.lobstr.io/v1/delivery \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: <content-type>' \
  --data '
{
  "webhook_fields.url": "<string>",
  "webhook_fields.is_active": true,
  "webhook_fields.retry": true,
  "webhook_fields.events.run.running": true,
  "webhook_fields.events.run.paused": true,
  "webhook_fields.events.run.done": true,
  "webhook_fields.events.run.error": true
}
'

import requests

url = "https://api.lobstr.io/v1/delivery"

payload = {
    "webhook_fields.url": "<string>",
    "webhook_fields.is_active": True,
    "webhook_fields.retry": True,
    "webhook_fields.events.run.running": True,
    "webhook_fields.events.run.paused": True,
    "webhook_fields.events.run.done": True,
    "webhook_fields.events.run.error": True
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "<content-type>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
  body: JSON.stringify({
    'webhook_fields.url': '<string>',
    'webhook_fields.is_active': true,
    'webhook_fields.retry': true,
    'webhook_fields.events.run.running': true,
    'webhook_fields.events.run.paused': true,
    'webhook_fields.events.run.done': true,
    'webhook_fields.events.run.error': true
  })
};

fetch('https://api.lobstr.io/v1/delivery', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/delivery",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'webhook_fields.url' => '<string>',
    'webhook_fields.is_active' => true,
    'webhook_fields.retry' => true,
    'webhook_fields.events.run.running' => true,
    'webhook_fields.events.run.paused' => true,
    'webhook_fields.events.run.done' => true,
    'webhook_fields.events.run.error' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: <content-type>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/delivery"

	payload := strings.NewReader("{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "<content-type>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/delivery")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "<content-type>")
  .body("{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/delivery")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n  \"webhook_fields.url\": \"<string>\",\n  \"webhook_fields.is_active\": true,\n  \"webhook_fields.retry\": true,\n  \"webhook_fields.events.run.running\": true,\n  \"webhook_fields.events.run.paused\": true,\n  \"webhook_fields.events.run.done\": true,\n  \"webhook_fields.events.run.error\": true\n}"

response = http.request(request)
puts response.read_body

{
  "webhook_fields.url": "<string>",
  "webhook_fields.is_active": true,
  "webhook_fields.retry": true,
  "webhook_fields.events.run.running": true,
  "webhook_fields.events.run.paused": true,
  "webhook_fields.events.run.done": true,
  "webhook_fields.events.run.error": true
}

Configure webhooks to receive real-time HTTP POST notifications when your squid’s events occur. Subscribe to specific events like run start, completion, pause, or errors.

Event Types

Event	Description	Trigger Condition
run.running	Run is actively executing	Emitted when a Squid run begins or resumes
run.paused	Run paused	Emitted when the run is temporarily halted (e.g., account limits reached)
run.done	Run completed successfully	Emitted when the run finishes without errors
run.error	Unexpected error occurred	When a run has crashed with an error

Webhook Payload

Lobstr sends a JSON payload to your webhook URL:

{
  "id": "c39b1922a5424c3c84a7cb2ec731bc5f",
  "object": "run",
  "event": "run.done",
  "squid": {
    "id": "8177beb16fdf4e0e8d4b2ba767d1ffb5",
    "name": "Google Maps Search Export (1)"
  },
  "timestamp": "2025/06/18 17:32:31"
}

Payload Fields

Field	Type	Description
id	string	Unique identifier for the run (hash)
object	string	Always “run”
event	string	One of the subscribed Event Types
squid.id	string	Squid hash
squid.name	string	Human-friendly Squid name
timestamp	string	Event timestamp in YYYY/MM/DD HH:MM:SS format (UTC)

Retry Mechanism

When retry is enabled:

Failed webhook deliveries are retried up to 3 times
Retries occur with a 15-minute delay between attempts

Webhook Response Requirements

Your endpoint should respond within 30 seconds
Return HTTP 200, 201, or 202 to indicate successful receipt

Headers

string

required

Your API authentication token. Value: Token YOUR_API_KEY

string

required

Must be application/json. Value: application/json

Query Parameters

string

required

The unique identifier (hash) of the squid for which to configure webhook delivery. Example: c106a44a98044ef18acc59986ae10967

Request Body

string

required

Your webhook endpoint URL that will receive POST requests. Example: "https://your-webhook.com/endpoint"

boolean

required

Master switch to enable/disable webhooks. Example: true

boolean

required

Enable automatic retries for failed webhook deliveries (up to 3 attempts with 15-minute delays). Example: false

boolean

required

Subscribe to run start/resume events. Example: true

boolean

required

Subscribe to run pause events. Example: false

boolean

required

Subscribe to run completion events. Example: false

boolean

required

Subscribe to run error events. Example: true

Response Field Explanations

string

Configured webhook endpoint URL. Example: "https://your-webhook.com/endpoint"

boolean

Whether webhooks are active. Example: true

boolean

Whether retry mechanism is enabled. Example: false

boolean

Subscription status for run.running events. Example: true

boolean

Subscription status for run.paused events. Example: false

boolean

Subscription status for run.done events. Example: false

boolean

Subscription status for run.error events. Example: true

Subscribe only to events you need. For most use cases, run.done and run.error are sufficient.

Your webhook endpoint must respond within 30 seconds or the delivery will be marked as failed.

Enable retry for production webhooks to handle temporary network issues or server downtime automatically.

Webhook payloads are sent as JSON with Content-Type: application/json header. Parse the request body to extract event data.

Use the Test Webhook endpoint to verify your endpoint is accessible and responding correctly before activating webhooks.

Implement proper authentication/validation in your webhook endpoint to ensure requests are coming from lobstr.io.

Code Examples

curl -X POST "https://api.lobstr.io/v1/delivery?squid=YOUR_SQUID_HASH" \
  -H "Authorization: Token YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "webhook_fields": {
      "url": "https://your-webhook.com/endpoint",
      "is_active": true,
      "retry": false,
      "events": {
        "run.running": true,
        "run.paused": false,
        "run.done": false,
        "run.error": true
      }
    }
  }'

import requests

url = "https://api.lobstr.io/v1/delivery"
headers = {
    "Authorization": "Token YOUR_API_KEY",
    "Content-Type": "application/json"
}
params = {
    "squid": "YOUR_SQUID_HASH"
}
payload = {
    "webhook_fields": {
        "url": "https://your-webhook.com/endpoint",
        "is_active": True,
        "retry": False,
        "events": {
            "run.running": True,
            "run.paused": False,
            "run.done": False,
            "run.error": True
        }
    }
}

response = requests.post(url, headers=headers, params=params, json=payload)
result = response.json()

print("Webhook delivery configured successfully!")
print(f"Webhook URL: {result['webhook_fields']['url']}")
print(f"Active: {result['webhook_fields']['is_active']}")
print(f"Retry enabled: {result['webhook_fields']['retry']}")
print("\nSubscribed events:")
for event, subscribed in result['webhook_fields']['events'].items():
    print(f"  {event}: {subscribed}")

Response

201

{
  "webhook_fields": {
    "url": "https://your-webhook.com/endpoint",
    "is_active": true,
    "retry": false,
    "events": {
      "run.running": false,
      "run.paused": true,
      "run.done": false,
      "run.error": true
    }
  }
}

Configure Amazon S3 Delivery Test Delivery Configuration

Overview

Getting Started

Core Concepts

User

Crawler

Squid

Delivery

Account

Task

Run

Result

Configure Webhook Delivery

Event Types

Webhook Payload

Payload Fields

Retry Mechanism

Webhook Response Requirements

Headers

Query Parameters

Request Body

Response Field Explanations

Code Examples

Response

​Event Types

​Webhook Payload

​Payload Fields

​Retry Mechanism

​Webhook Response Requirements

​Headers

​Query Parameters

​Request Body

​Response Field Explanations

​Code Examples

​Response

Event Types

Webhook Payload

Payload Fields

Retry Mechanism

Webhook Response Requirements

Headers

Query Parameters

Request Body

Response Field Explanations

Code Examples

Response