Fetch Agent

Starter Kit - Ships with your template. You own it - modify freely.

Basic Usage

agents:
  - name: fetch
    agent: fetch
    inputs:
      url: https://api.example.com/data

Output:

{
  "success": true,
  "status": 200,
  "statusText": "OK",
  "headers": {...},
  "body": {...},
  "duration": 150,
  "attempt": 1
}

Inputs

inputs:
  url:
    type: string
    format: uri
    required: true
    description: URL to fetch

  body:
    type: any
    description: Request body (for POST, PUT, PATCH)

  headers:
    type: object
    description: Additional request headers

Configuration

config:
  method:
    type: string
    enum: [GET, POST, PUT, DELETE, PATCH]
    default: GET
    description: HTTP method

  headers:
    type: object
    description: Default headers for all requests

  retry:
    type: integer
    default: 3
    description: Number of retry attempts

  timeout:
    type: integer
    default: 30000
    description: Request timeout in milliseconds

  retryDelay:
    type: integer
    default: 1000
    description: Initial delay for exponential backoff (ms)

Examples

GET Request

agents:
  - name: fetch
    agent: fetch
    inputs:
      url: https://api.example.com/users/${input.user_id}

POST Request

agents:
  - name: create
    agent: fetch
    inputs:
      url: https://api.example.com/users
      body:
        name: ${input.name}
        email: ${input.email}
      headers:
        Authorization: Bearer ${env.API_KEY}
    config:
      method: POST

With Retries

agents:
  - name: fetch
    agent: fetch
    inputs:
      url: https://api.example.com/data
    config:
      retry: 5
      retryDelay: 1000

With Timeout

agents:
  - name: fetch
    agent: fetch
    inputs:
      url: https://api.example.com/slow-endpoint
    config:
      timeout: 5000  # 5 seconds

Advanced Patterns

Multiple Endpoints

agents:
  # Fetch in parallel
  - name: fetch-users
    agent: fetch
    inputs:
      url: https://api.example.com/users

  - name: fetch-posts
    agent: fetch
    inputs:
      url: https://api.example.com/posts

  - name: fetch-comments
    agent: fetch
    inputs:
      url: https://api.example.com/comments

  # Combine results
  - name: combine
    operation: code
    config:
      script: scripts/combine-api-results
    input:
      users: ${fetch-users.output.body}
      posts: ${fetch-posts.output.body}
      comments: ${fetch-comments.output.body}

// scripts/combine-api-results.ts
import type { AgentExecutionContext } from '@ensemble-edge/conductor'

export default function combineApiResults(context: AgentExecutionContext) {
  const { users, posts, comments } = context.input

  return {
    users,
    posts,
    comments
  }
}

Primary + Fallback

agents:
  - name: fetch-primary
    agent: fetch
    inputs:
      url: https://primary-api.example.com/data
    config:
      retry: 2

  - name: fetch-secondary
    condition: ${fetch-primary.failed}
    agent: fetch
    inputs:
      url: https://secondary-api.example.com/data

  - name: result
    operation: code
    config:
      script: scripts/select-fetch-result
    input:
      primary_output: ${fetch-primary.output}
      secondary_output: ${fetch-secondary.output}

// scripts/select-fetch-result.ts
import type { AgentExecutionContext } from '@ensemble-edge/conductor'

export default function selectFetchResult(context: AgentExecutionContext) {
  const { primary_output, secondary_output } = context.input
  return primary_output || secondary_output
}

Paginated Requests

agents:
  - name: fetch-page-1
    agent: fetch
    inputs:
      url: https://api.example.com/items?page=1

  - name: fetch-page-2
    condition: ${fetch-page-1.output.body.hasMore}
    agent: fetch
    inputs:
      url: https://api.example.com/items?page=2

  - name: fetch-page-3
    condition: ${fetch-page-2.output.body.hasMore}
    agent: fetch
    inputs:
      url: https://api.example.com/items?page=3

  - name: combine
    operation: code
    config:
      script: scripts/combine-paginated-results
    input:
      page1_items: ${fetch-page-1.output.body.items}
      page2_items: ${fetch-page-2.output.body.items || []}
      page3_items: ${fetch-page-3.output.body.items || []}

// scripts/combine-paginated-results.ts
import type { AgentExecutionContext } from '@ensemble-edge/conductor'

export default function combinePaginatedResults(context: AgentExecutionContext) {
  const { page1_items, page2_items, page3_items } = context.input

  return {
    items: [
      ...page1_items,
      ...page2_items,
      ...page3_items
    ]
  }
}

Authenticated Requests

agents:
  # Get auth token
  - name: auth
    agent: fetch
    inputs:
      url: https://auth.example.com/token
      body:
        client_id: ${env.CLIENT_ID}
        client_secret: ${env.CLIENT_SECRET}
    config:
      method: POST
    cache:
      ttl: 3600
      key: auth-token

  # Use token
  - name: fetch-data
    agent: fetch
    inputs:
      url: https://api.example.com/data
      headers:
        Authorization: Bearer ${auth.output.body.access_token}

Conditional Requests

agents:
  - name: check-cache
    operation: storage
    config:
      type: kv
      action: get
      key: data-${input.id}

  - name: fetch-if-stale
    condition: ${!check-cache.output}
    agent: fetch
    inputs:
      url: https://api.example.com/data/${input.id}

  - name: update-cache
    condition: ${fetch-if-stale.executed}
    operation: storage
    config:
      type: kv
      action: put
      key: data-${input.id}
      value: ${fetch-if-stale.output.body}
      ttl: 3600

Output Schema

{
  success: boolean;        // Whether request succeeded
  status: number;          // HTTP status code
  statusText: string;      // HTTP status text
  headers: object;         // Response headers
  body: any;              // Response body (parsed JSON if applicable)
  duration: number;       // Total request time in ms
  attempt: number;        // Which retry attempt succeeded (1-based)
}

Error Handling

agents:
  - name: fetch
    agent: fetch
    inputs:
      url: ${input.url}

  - name: handle-error
    condition: ${fetch.failed}
    operation: code
    config:
      script: scripts/handle-fetch-error
    input:
      error: ${fetch.error}

// scripts/handle-fetch-error.ts
import type { AgentExecutionContext } from '@ensemble-edge/conductor'

export default function handleFetchError(context: AgentExecutionContext) {
  const { error } = context.input

  if (error.status === 404) {
    return { notFound: true }
  } else if (error.status >= 500) {
    return { serverError: true, retryable: true }
  } else if (error.timeout) {
    return { timeout: true, retryable: true }
  } else {
    return { error: error.message }
  }
}

Best Practices

1. Set Appropriate Timeouts

config:
  timeout: 5000   # 5s for fast APIs
  timeout: 30000  # 30s for slow APIs (default)
  timeout: 60000  # 60s for webhooks

2. Configure Retries with Backoff

config:
  retry: 5
  retryDelay: 1000  # Exponential backoff starting at 1s

3. Secure API Keys

headers:
  Authorization: Bearer ${env.API_KEY}  # Use env vars

4. Validate Responses

agents:
  - name: fetch
    agent: fetch

  - name: validate
    agent: validate
    inputs:
      data: ${fetch.output.body}
      schema: ${component.response-schema@v1.0.0}

5. Use Cache for Expensive Requests

agents:
  - name: fetch
    agent: fetch
    cache:
      ttl: 3600  # 1 hour
      key: api-${input.endpoint}

Common Use Cases

External API Integration

agents:
  - name: fetch-weather
    agent: fetch
    inputs:
      url: https://api.weather.com/v1/current
      headers:
        X-API-Key: ${env.WEATHER_API_KEY}
    cache:
      ttl: 1800  # 30 minutes

Webhook Proxy

ensemble: webhook-proxy

agents:
  - name: validate
    agent: validate
    inputs:
      data: ${input}
      schema: ${component.webhook-schema@v1.0.0}

  - name: forward
    condition: ${validate.output.valid}
    agent: fetch
    inputs:
      url: ${env.WEBHOOK_TARGET}
      body: ${validate.output.data}
    config:
      method: POST
      timeout: 60000

API Aggregator

ensemble: aggregate-data

agents:
  # Fetch from multiple sources
  - name: fetch-source-1
    agent: fetch
    inputs:
      url: https://api1.example.com/data

  - name: fetch-source-2
    agent: fetch
    inputs:
      url: https://api2.example.com/data

  - name: fetch-source-3
    agent: fetch
    inputs:
      url: https://api3.example.com/data

  # Merge results
  - name: merge
    operation: code
    config:
      script: scripts/merge-api-sources
    input:
      source1: ${fetch-source-1.output.body}
      source2: ${fetch-source-2.output.body}
      source3: ${fetch-source-3.output.body}

// scripts/merge-api-sources.ts
import type { AgentExecutionContext } from '@ensemble-edge/conductor'

export default function mergeApiSources(context: AgentExecutionContext) {
  const { source1, source2, source3 } = context.input

  return {
    combined: {
      source1,
      source2,
      source3
    }
  }
}

GraphQL Query

agents:
  - name: query
    agent: fetch
    inputs:
      url: https://api.example.com/graphql
      body:
        query: |
          query GetUser($id: ID!) {
            user(id: $id) {
              name
              email
              posts {
                title
                content
              }
            }
          }
        variables:
          id: ${input.user_id}
      headers:
        Content-Type: application/json
    config:
      method: POST

vs http Operation

Use fetch agent for:

Built-in retry logic with exponential backoff
Automatic error handling
Common patterns

Use http operation for:

Custom retry logic
Streaming responses
Low-level control

Example comparison:

# fetch agent (higher level)
agents:
  - name: fetch
    agent: fetch
    inputs:
      url: ${url}
    config:
      retry: 3
      timeout: 5000

# http operation (lower level)
agents:
  - name: fetch
    operation: http
    config:
      url: ${url}
      timeout: 5000
    retry:
      maxAttempts: 3

Next Steps

http

HTTP operation reference

scrape

Web scraping

validate

Data validation

Starter Kit Overview

All starter kit agents

Conductor

Getting Started

Core Concepts

Building

Components

Operations Reference

Plugins

Starter Kit

Playbooks

Reference

Basic Usage

Inputs

Configuration

Examples

GET Request

POST Request

With Retries

With Timeout

Advanced Patterns

Multiple Endpoints

Primary + Fallback

Paginated Requests

Authenticated Requests

Conditional Requests

Output Schema

Error Handling

Best Practices

Common Use Cases

External API Integration

Webhook Proxy

API Aggregator

GraphQL Query

vs http Operation

Next Steps

http

scrape

validate

Starter Kit Overview

​Basic Usage

​Inputs

​Configuration

​Examples

​GET Request

​POST Request

​With Retries

​With Timeout

​Advanced Patterns

​Multiple Endpoints

​Primary + Fallback

​Paginated Requests

​Authenticated Requests

​Conditional Requests

​Output Schema

​Error Handling

​Best Practices

​Common Use Cases

​External API Integration

​Webhook Proxy

​API Aggregator

​GraphQL Query

​vs http Operation

​Next Steps

http

scrape

validate

Starter Kit Overview

Basic Usage

Inputs

Configuration

Examples

GET Request

POST Request

With Retries

With Timeout

Advanced Patterns

Multiple Endpoints

Primary + Fallback

Paginated Requests

Authenticated Requests

Conditional Requests

Output Schema

Error Handling

Best Practices

Common Use Cases

External API Integration

Webhook Proxy

API Aggregator

GraphQL Query

vs http Operation

Next Steps