Skip to main content
When alerts are triggered or resolved in your AgentMark platform, it sends notifications to your webhook endpoint. This allows you to integrate with your monitoring systems and take appropriate actions.
Notification is only triggered for alerts for which the webhook is configured.

Processing Alert Events

if (event.type === "alert") {
  const alertData = event.data;
  console.log('Alert received:', alertData);
  
  try {
    // Process alert based on status and type
    if (alertData.alert.status === "triggered") {
      await handleTriggeredAlert(alertData);
    } else if (alertData.alert.status === "resolved") {
      await handleResolvedAlert(alertData);
    }
    
    return NextResponse.json({ 
      message: "Alert processed successfully",
      alertId: alertData.alert.id,
      status: alertData.alert.status 
    });
  } catch (error) {
    console.error("Alert processing error:", error);
    return NextResponse.json(
      { 
        message: "Error processing alert",
        error: error.message 
      },
      { status: 500 }
    );
  }
}

async function handleTriggeredAlert(alertData: any) {
  const { alert, message, timestamp } = alertData;
  
  // Handle different types of triggered alerts
  switch (alert.type) {
    case "cost":
      await handleCostAlert(alert, message, timestamp);
      break;
    case "latency":
      await handleLatencyAlert(alert, message, timestamp);
      break;
    case "error_rate":
      await handleErrorRateAlert(alert, message, timestamp);
      break;
    default:
      console.warn(`Unknown alert type: ${alert.type}`);
  }
}

async function handleResolvedAlert(alertData: any) {
  const { alert, message, timestamp } = alertData;
  
  console.log(`Alert ${alert.id} resolved at ${timestamp}`);
  
  // Notify that the issue has been resolved
  // You could send notifications, update dashboards, etc.
}

async function handleCostAlert(alert: any, message: string, timestamp: string) {
  console.log(`Cost alert: Spending ${alert.currentValue} exceeded threshold ${alert.threshold}`);
  
  // Example: Send notification to Slack, email, etc.
  // await sendSlackNotification({
  //   text: `🚨 Cost Alert: ${message}`,
  //   details: {
  //     current: alert.currentValue,
  //     threshold: alert.threshold,
  //     timeWindow: alert.timeWindow
  //   }
  // });
}

async function handleLatencyAlert(alert: any, message: string, timestamp: string) {
  console.log(`Latency alert: Response time ${alert.currentValue}ms exceeded threshold ${alert.threshold}ms`);
  
  // Example: Auto-scale resources, investigate performance issues
  // await autoScaleResources();
  // await createPerformanceTicket(alert);
}

async function handleErrorRateAlert(alert: any, message: string, timestamp: string) {
  console.log(`Error rate alert: ${alert.currentValue}% errors exceeded threshold ${alert.threshold}%`);
  
  // Example: Trigger incident response, roll back deployment
  // await triggerIncidentResponse(alert);
  // await checkRecentDeployments();
}

Event Format

{
  "event": {
    "type": "alert",
    "data": {
      "alert": {
        "id": string,
        "currentValue": number,
        "threshold": number,
        "status": "triggered | resolved",
        "timeWindow": string,
        "type": "cost | latency | error_rate"
      },
      "message": string,
      "timestamp": string
    }
  }
}

Alert Types

AgentMark supports three types of alerts:
  1. Cost Alerts
    • Monitor API usage costs
    • Set thresholds for spending
    • Track cost trends
  2. Latency Alerts
    • Monitor response times
    • Set performance thresholds
    • Track latency issues
  3. Error Rate Alerts
    • Monitor error frequencies
    • Set error rate thresholds
    • Track system reliability

Integration Examples

Slack Notifications

async function sendSlackNotification(alert: any, message: string) {
  const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;
  
  if (!slackWebhookUrl) return;
  
  const payload = {
    text: `🚨 AgentMark Alert: ${alert.type.toUpperCase()}`,
    blocks: [
      {
        type: "section",
        text: {
          type: "mrkdwn",
          text: message,
        },
      },
      {
        type: "section",
        fields: [
          {
            type: "mrkdwn",
            text: `*Alert ID:*\n${alert.id}`,
          },
          {
            type: "mrkdwn",
            text: `*Status:*\n${alert.status}`,
          },
          {
            type: "mrkdwn",
            text: `*Current Value:*\n${alert.currentValue}`,
          },
          {
            type: "mrkdwn",
            text: `*Threshold:*\n${alert.threshold}`,
          },
        ],
      },
    ],
  };
  
  await fetch(slackWebhookUrl, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(payload),
  });
}

Email Notifications

async function sendEmailAlert(alert: any, message: string) {
  const emailConfig = {
    from: process.env.ALERT_EMAIL_FROM,
    to: process.env.ALERT_EMAIL_TO,
    subject: `AgentMark Alert: ${alert.type} - ${alert.status}`,
    html: `
      <h2>AgentMark Alert</h2>
      <p>${message}</p>
      <table>
        <tr><td><strong>Alert ID:</strong></td><td>${alert.id}</td></tr>
        <tr><td><strong>Type:</strong></td><td>${alert.type}</td></tr>
        <tr><td><strong>Status:</strong></td><td>${alert.status}</td></tr>
        <tr><td><strong>Current Value:</strong></td><td>${alert.currentValue}</td></tr>
        <tr><td><strong>Threshold:</strong></td><td>${alert.threshold}</td></tr>
        <tr><td><strong>Time Window:</strong></td><td>${alert.timeWindow}</td></tr>
      </table>
    `,
  };
  
  // Use your preferred email service (SendGrid, AWS SES, etc.)
  // await sendEmail(emailConfig);
}

Best Practices

  1. Immediate Response
    • Process alerts quickly to minimize impact
    • Return appropriate HTTP status codes
    • Log all alert events for auditing
  2. Error Handling
    • Always wrap alert processing in try-catch blocks
    • Provide meaningful error messages
    • Implement retry logic for failed notifications
  3. Notification Routing
    • Route different alert types to appropriate channels
    • Implement escalation policies for critical alerts
    • Avoid notification fatigue with intelligent filtering
  4. Monitoring Integration
    • Forward alerts to your monitoring systems
    • Create dashboard visualizations for alert trends
    • Set up automated responses for common scenarios

Have Questions?

We’re here to help! Choose the best way to reach us: