Configure Amazon S3 Delivery

curl --request POST \
  --url https://api.lobstr.io/v1/delivery \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: <content-type>' \
  --data '
{
  "s3_fields.bucket": "<string>",
  "s3_fields.is_active": true,
  "s3_fields.target_path": "<string>",
  "s3_fields.aws_access_key": "<string>",
  "s3_fields.aws_secret_key": "<string>"
}
'

import requests

url = "https://api.lobstr.io/v1/delivery"

payload = {
    "s3_fields.bucket": "<string>",
    "s3_fields.is_active": True,
    "s3_fields.target_path": "<string>",
    "s3_fields.aws_access_key": "<string>",
    "s3_fields.aws_secret_key": "<string>"
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "<content-type>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
  body: JSON.stringify({
    's3_fields.bucket': '<string>',
    's3_fields.is_active': true,
    's3_fields.target_path': '<string>',
    's3_fields.aws_access_key': '<string>',
    's3_fields.aws_secret_key': '<string>'
  })
};

fetch('https://api.lobstr.io/v1/delivery', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/delivery",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    's3_fields.bucket' => '<string>',
    's3_fields.is_active' => true,
    's3_fields.target_path' => '<string>',
    's3_fields.aws_access_key' => '<string>',
    's3_fields.aws_secret_key' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: <content-type>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/delivery"

	payload := strings.NewReader("{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "<content-type>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/delivery")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "<content-type>")
  .body("{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/delivery")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "s3_fields.bucket": "<string>",
  "s3_fields.is_active": true,
  "s3_fields.target_path": "<string>",
  "s3_fields.aws_access_key": "<string>",
  "s3_fields.aws_secret_key": "<string>"
}

POST

delivery

Configure Amazon S3 Delivery

curl --request POST \
  --url https://api.lobstr.io/v1/delivery \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: <content-type>' \
  --data '
{
  "s3_fields.bucket": "<string>",
  "s3_fields.is_active": true,
  "s3_fields.target_path": "<string>",
  "s3_fields.aws_access_key": "<string>",
  "s3_fields.aws_secret_key": "<string>"
}
'

import requests

url = "https://api.lobstr.io/v1/delivery"

payload = {
    "s3_fields.bucket": "<string>",
    "s3_fields.is_active": True,
    "s3_fields.target_path": "<string>",
    "s3_fields.aws_access_key": "<string>",
    "s3_fields.aws_secret_key": "<string>"
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "<content-type>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
  body: JSON.stringify({
    's3_fields.bucket': '<string>',
    's3_fields.is_active': true,
    's3_fields.target_path': '<string>',
    's3_fields.aws_access_key': '<string>',
    's3_fields.aws_secret_key': '<string>'
  })
};

fetch('https://api.lobstr.io/v1/delivery', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.lobstr.io/v1/delivery",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    's3_fields.bucket' => '<string>',
    's3_fields.is_active' => true,
    's3_fields.target_path' => '<string>',
    's3_fields.aws_access_key' => '<string>',
    's3_fields.aws_secret_key' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: <content-type>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.lobstr.io/v1/delivery"

	payload := strings.NewReader("{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "<content-type>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/delivery")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "<content-type>")
  .body("{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.lobstr.io/v1/delivery")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n  \"s3_fields.bucket\": \"<string>\",\n  \"s3_fields.is_active\": true,\n  \"s3_fields.target_path\": \"<string>\",\n  \"s3_fields.aws_access_key\": \"<string>\",\n  \"s3_fields.aws_secret_key\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "s3_fields.bucket": "<string>",
  "s3_fields.is_active": true,
  "s3_fields.target_path": "<string>",
  "s3_fields.aws_access_key": "<string>",
  "s3_fields.aws_secret_key": "<string>"
}

Configure Amazon S3 delivery to export run results directly to an AWS S3 bucket. Results will be automatically uploaded to your specified bucket and path when runs complete.

Authentication Options

You can authenticate with S3 in two ways:

AWS Credentials: Provide aws_access_key and aws_secret_key in the request
Bucket Policy: If you don’t provide credentials, ensure your bucket policy allows public WRITE permissions

Headers

string

required

Your API authentication token. Value: Token YOUR_API_KEY

string

required

Must be application/json. Value: application/json

Query Parameters

string

required

The unique identifier (hash) of the squid for which to configure S3 delivery. Example: c106a44a98044ef18acc59986ae10967

Request Body

string

required

Name of the S3 bucket where data will be stored. Example: "my-bucket"

boolean

required

Set to true to enable S3 delivery, false to disable. Example: true

string

required

Folder path inside the bucket where results will be saved. Example: "exports"

string

(Optional) AWS Access Key for authentication. If not provided, bucket must have public WRITE permissions. Example: "AKIAIOSFODNN7EXAMPLE"

string

(Optional) AWS Secret Key for authentication. If not provided, bucket must have public WRITE permissions. Example: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"

Response Field Explanations

string

S3 bucket name. Example: "my-bucket"

boolean

Whether S3 delivery is active. Example: true

string

Target folder path in bucket. Example: "exports"

string

AWS Access Key (returned if provided). Example: "AKIAIOSFODNN7EXAMPLE"

string

AWS Secret Key (returned if provided). Example: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"

If you don’t provide AWS credentials, your bucket policy MUST allow public WRITE permissions for Lobstr to upload files.

For better security, provide AWS credentials instead of making your bucket publicly writable. Use IAM users with limited permissions.

The target_path creates a folder structure in your bucket. Results will be uploaded to s3://bucket/target_path/filename.csv

Use the Test S3 endpoint to verify your bucket permissions and credentials before activating automatic delivery.

AWS credentials are stored and returned in API responses. Ensure you’re using HTTPS and handle responses securely.

Code Examples

curl -X POST "https://api.lobstr.io/v1/delivery?squid=YOUR_SQUID_HASH" \
  -H "Authorization: Token YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "s3_fields": {
      "bucket": "my-bucket",
      "is_active": true,
      "target_path": "exports",
      "aws_access_key": "your_access_key",
      "aws_secret_key": "your_secret_key"
    }
  }'

import requests

url = "https://api.lobstr.io/v1/delivery"
headers = {
    "Authorization": "Token YOUR_API_KEY",
    "Content-Type": "application/json"
}
params = {
    "squid": "YOUR_SQUID_HASH"
}
payload = {
    "s3_fields": {
        "bucket": "my-bucket",
        "is_active": True,
        "target_path": "exports",
        "aws_access_key": "your_access_key",
        "aws_secret_key": "your_secret_key"
    }
}

response = requests.post(url, headers=headers, params=params, json=payload)
result = response.json()

print("Amazon S3 delivery configured successfully!")
print(f"Bucket: {result['s3_fields']['bucket']}")
print(f"Target path: {result['s3_fields']['target_path']}")
print(f"Active: {result['s3_fields']['is_active']}")

Response

201

{
  "s3_fields": {
    "bucket": "my-bucket",
    "is_active": true,
    "target_path": "exports",
    "aws_access_key": "your_access_key",
    "aws_secret_key": "your_secret_key"
  }
}

Configure Google Sheet Delivery Configure Webhook Delivery

Overview

Getting Started

Core Concepts

User

Crawler

Squid

Delivery

Account

Task

Run

Result

Configure Amazon S3 Delivery

Authentication Options

Headers

Query Parameters

Request Body

Response Field Explanations

Code Examples

Response

​Authentication Options

​Headers

​Query Parameters

​Request Body

​Response Field Explanations

​Code Examples

​Response

Authentication Options

Headers

Query Parameters

Request Body

Response Field Explanations

Code Examples

Response