Add Tasks
curl --request POST \
--url https://api.lobstr.io/v1/tasks \
--header 'Authorization: <authorization>' \
--header 'Content-Type: <content-type>' \
--data '
{
"squid": "<string>",
"tasks": [
{}
]
}
'import requests
url = "https://api.lobstr.io/v1/tasks"
payload = {
"squid": "<string>",
"tasks": [{}]
}
headers = {
"Authorization": "<authorization>",
"Content-Type": "<content-type>"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
body: JSON.stringify({squid: '<string>', tasks: [{}]})
};
fetch('https://api.lobstr.io/v1/tasks', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/tasks",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'squid' => '<string>',
'tasks' => [
[
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>",
"Content-Type: <content-type>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/tasks"
payload := strings.NewReader("{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<authorization>")
req.Header.Add("Content-Type", "<content-type>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/tasks")
.header("Authorization", "<authorization>")
.header("Content-Type", "<content-type>")
.body("{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/tasks")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}"
response = http.request(request)
puts response.read_body{
"duplicated_count": 123,
"tasks": [
{}
],
"tasks[].id": "<string>",
"tasks[].object": "<string>",
"tasks[].created_at": "<string>",
"tasks[].is_active": true,
"tasks[].params": {},
"tasks[].module": "<string>"
}Task
Add Tasks
Create new tasks within a specified squid
POST
/
v1
/
tasks
Add Tasks
curl --request POST \
--url https://api.lobstr.io/v1/tasks \
--header 'Authorization: <authorization>' \
--header 'Content-Type: <content-type>' \
--data '
{
"squid": "<string>",
"tasks": [
{}
]
}
'import requests
url = "https://api.lobstr.io/v1/tasks"
payload = {
"squid": "<string>",
"tasks": [{}]
}
headers = {
"Authorization": "<authorization>",
"Content-Type": "<content-type>"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<authorization>', 'Content-Type': '<content-type>'},
body: JSON.stringify({squid: '<string>', tasks: [{}]})
};
fetch('https://api.lobstr.io/v1/tasks', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.lobstr.io/v1/tasks",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'squid' => '<string>',
'tasks' => [
[
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: <authorization>",
"Content-Type: <content-type>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.lobstr.io/v1/tasks"
payload := strings.NewReader("{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<authorization>")
req.Header.Add("Content-Type", "<content-type>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.lobstr.io/v1/tasks")
.header("Authorization", "<authorization>")
.header("Content-Type", "<content-type>")
.body("{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.lobstr.io/v1/tasks")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = '<content-type>'
request.body = "{\n \"squid\": \"<string>\",\n \"tasks\": [\n {}\n ]\n}"
response = http.request(request)
puts response.read_body{
"duplicated_count": 123,
"tasks": [
{}
],
"tasks[].id": "<string>",
"tasks[].object": "<string>",
"tasks[].created_at": "<string>",
"tasks[].is_active": true,
"tasks[].params": {},
"tasks[].module": "<string>"
}This endpoint creates new tasks within a specified squid. Tasks define the specific scraping jobs to be executed, such as URLs to scrape or search parameters.
Note: The accepted task input keys (e.g.,
url, city, activity) depend on the crawler used in your squid. Use the Get Crawler Parameters endpoint to discover valid keys for your crawler.
Headers
string
required
Your API authentication token. Value:
Token YOUR_API_KEYstring
required
Must be application/json. Value:
application/jsonRequest Body
string
required
The hash ID of the squid to add tasks to. Example:
"a1b2c3d4e5f6g7h8i9j0"array
required
Array of task objects. Each task contains parameters specific to the crawler. Example:
[{"url": "https://example.com"}]Response Field Explanations
integer
Number of tasks that were duplicates and not added. Example:
0array
Array of created task objects.
string
Unique task identifier (hash). Example:
"c5e29d2aba8b77cdc56391e7405302de"string
Always
"task".string
Task creation timestamp (ISO 8601). Example:
"2025-02-10T14:19:32.916901"boolean
Whether the task is active and ready to run. Example:
trueobject
Task parameters specific to the crawler (URL, search terms, etc.). Example:
{"url": "https://..."}string
Identifier of the crawler module this task belongs to.
Use the Get Crawler Parameters endpoint to discover which task keys are valid for your specific crawler.
Duplicate tasks (same parameters) are automatically detected and not added. Check duplicated_count in the response.
Task parameters vary by crawler. Using invalid keys will result in task creation failure.
For bulk task creation, consider using the Upload Tasks endpoint with a CSV file instead.
Code Examples
curl -X POST "https://api.lobstr.io/v1/tasks" \
-H "Authorization: Token YOUR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"squid": "a1b2c3d4e5f6g7h8i9j0",
"tasks": [
{"url": "https://www.linkedin.com/in/johndoe"},
{"url": "https://www.linkedin.com/in/janedoe"}
]
}'
import requests
url = "https://api.lobstr.io/v1/tasks"
headers = {
"Authorization": "Token YOUR_API_KEY",
"Content-Type": "application/json"
}
# Define tasks to add to the squid
payload = {
"squid": "a1b2c3d4e5f6g7h8i9j0",
"tasks": [
{"url": "https://www.linkedin.com/in/johndoe"},
{"url": "https://www.linkedin.com/in/janedoe"}
]
}
response = requests.post(url, headers=headers, json=payload)
data = response.json()
print(f"Added {len(data['tasks'])} tasks")
print(f"Duplicates skipped: {data['duplicated_count']}")
# Display created task IDs
for task in data['tasks']:
print(f" - {task['id']}")
Response
200
{
"duplicated_count": 0,
"tasks": [
{
"id": "6b9d3afdf4e76df374915a50d7a495c4",
"object": "task",
"created_at": "2025-02-10T14:19:32.916901",
"is_active": true,
"params": {"url": "https://www.linkedin.com/in/johndoe"},
"module": "4734d096159ef05210e0e1677e8be823"
},
{
"id": "c5e29d2aba8b77cdc56391e7405302de",
"object": "task",
"created_at": "2025-02-10T14:19:32.916901",
"is_active": true,
"params": {"url": "https://www.linkedin.com/in/janedoe"},
"module": "4734d096159ef05210e0e1677e8be823"
}
]
}
400
{
"error": {
"message": "Invalid task parameters. Each task must contain required fields for the crawler.",
"type": "validation_error",
"param": "tasks"
}
}