Agent Implementation

The customer support agent is implemented using LangGraph as a multi-agent system where a supervisor agent is responsible for routing each customer request to the right agent (customer agent for direct response to customer, database coodinator for database transactions, and knowledge agent for a RAG query to the company knowledge base).

Defining the contract specification

This specification is made up of 4 scenarios: each containing the data for the testbot to simulate a scenario along with the requirements defined in contracts.

define.py
from relari_otel.specifications import Specifications, Scenario, Contract, Precondition, Pathcondition, Postcondition, Level

refund_eligible = Scenario(
    uuid="sce-refund-1",
    name="Eligible Refund Request",
    data={
           "uid":"simulation-eligible-refund",
           "context":[{"text":"Customer wants to return a damaged sweater to store credit"}],
           "tasks":[
              {
                 "primary_task":"Request refund for damaged sweater. Don't ask questions about the process of return/refund, let the support agent guide you. Provide only one piece of information at a time.",
                 "additional_info":"Here're some information you may provide to the agent if asked:\nOrder number: order_01JGQF1YAYNFDV874MKFR33CAA\nYour email: jane.doe@example.com\nPhotos of damage: Image url: https://m.media-amazon.com/images/I/A1mN82gBRyL._CLa%7C2140%2C2000%7CA1u0zfdjH1L.png%7C0%2C0%2C2140%2C2000%2B0.0%2C0.0%2C2140.0%2C2000.0_AC_SX679_.png\nYou discovered the damage after a week after purchase.\n The damage is on the sleaves - there's a hole.\nYou haven't worn the sweater.\nYou want a refund to store credit.",
                 "expected_response":"Purchase was refunded, or if the issue was escalated to a manager."
              }
           ]
    },
    contracts=[
        Contract(
            uuid="con-475c011f",
            name="con-refund-1",
            requirements=[

                # Checks if the agent received the right inputs
                Precondition("Customer provides order number", level=Level.MUST),
                Precondition("Customer provides email address if asked", level=Level.SHOULD),
                Precondition("Customer should provided photo of damage to the sweater if asked", level=Level.SHOULD),

                # Defines the correct trajectory
                Pathcondition("Collect when the damage was first noticed, location of damage, and the photo of damage", level=Level.MUST),
                Pathcondition("Check against the return policy and determine that the order is eligible for a refund", level=Level.MUST),

                # Defines what the output should look like
                Postcondition("Provide the refund id to the customer", on="output", level=Level.MUST),
                Postcondition("Process refund request as a store credit", on="output", level=Level.MUST),
            ],
        )
    ],
)

refund_ineligible = Scenario(
    uuid="sce-refund-2",
    name="Ineligible Refund Request",
    data={
        "uid": "sim-ineligible-refund",
        "context": [
            {
            "text": "Customer wants to return a damaged sweater that was worn before damage (ineligible for refund)"
            }
        ],
        "tasks": [
            {
            "primary_task": "Request refund for damaged sweater. Don't ask questions about the process of return/refund, let the support agent guide you. Provide only one piece of information at a time.",
            "additional_info": "Here're some information you may provide to the agent if asked:\nOrder number: order_01JGQF1YAYNFDV874MKFR33CAA\nYour email: jane.doe@example.com\nPhotos of damage: Image url: https://m.media-amazon.com/images/I/A1mN82gBRyL._CLa%7C2140%2C2000%7CA1u0zfdjH1L.png%7C0%2C0%2C2140%2C2000%2B0.0%2C0.0%2C2140.0%2C2000.0_AC_SX679_.png\nYou discovered the damage after wearing it for 2 weeks.\nThe damage is on the sleeves - there's a tear.\nYou have worn the sweater multiple times.\nYou want a refund to original payment method.\nIf denied, you should argue for your case once and only once.",
            "expected_response": "Refund request denied or escalated to a manager."
            }
        ]
        },
    contracts=[
        Contract(
            uuid="con-52e81d70",
            name="con-refund-2",
            requirements=[
                Precondition("Customer provides or selects order number order_01JGQF1YAYNFDV874MKFR33CAA", level=Level.MUST),
                Precondition("Customer provides email address if asked", level=Level.SHOULD),
                Postcondition("Deny the refund request", on="output", level=Level.MUST),
                Postcondition("Escalate to a manager if the customer insists on a refund", on="output", level=Level.SHOULD),
                Pathcondition("Collect when the damage was first noticed, location of damage, and whether the item was worn", level=Level.MUST),
                Pathcondition("Check against the return policy and determine that the order is not eligible for a refund", level=Level.MUST),
            ],
        )
    ],
)

shipping_status = Scenario(
    uuid="sce-shipping-1",
    name="Shipping Status Inquiry",
    data={
        "uid": "sim-shipping-1",
        "context": [
            {
                "text": "Customer inquiring about shipping status"
            }
        ],
        "tasks": [
            {
                "primary_task": "Check shipping status for sweatshirt order order_01JGMMMPHCQM01FBAHY23FDA1",
                "additional_info": "Provide your email (jane.doe@example.com) when asked",
                "expected_response": "Should verify order details and say order doesn't exist"
            }
        ]
    },
    contracts=[
        Contract(
            uuid="con-shipping-1",
            name="con-shipping-1",
            requirements=[
                Precondition("Customer has provided order number order_01JGMMMPHCQM01FBAHY23FDA1", level=Level.MUST),
                Precondition("Customer has provided email address for verification", level=Level.SHOULD),
                Postcondition("Inform customer that the order number does not exist in the system", on="output", level=Level.MUST),
                Postcondition("Provides escalation contact information to customer", on="output", level=Level.SHOULD),
                Pathcondition("Use email address to retrieve associated orders, and verify the order number doesn't exist", level=Level.MUST),
                Pathcondition("Do NOT make up any shipping status, delivery dates, or any reason for the order not existing", level=Level.MUST),
                Pathcondition("Do NOT ask for unnecessary personal information beyond order number and email", level=Level.SHOULD),
            ],
        )
    ],
)

product_recommendation = Scenario(
    uuid="sce-product-1",
    name="Product Recommendation Request",
    data={
        "uid": "sim-product-1",
        "context": [
            {
                "text": "Customer ask for product recommendation"
            }
        ],
        "tasks": [
            {
                "primary_task": "Ask about what shirts are available (don't ask specifically about t-shirts)",
                "additional_info": "You want tshirts and not sweatshirts (don't provide this information unless asked).",
                "expected_response": "Should provide the details of the tshirts available."
            }
        ]
    },
    contracts=[
        Contract(
            uuid="con-product-1",
            name="con-product-1",
            requirements=[
                Precondition("Customer express interest in shirts first (not t-shirts specifically)", level=Level.MUST),
                Precondition("Customer specifies that they want t-shirts and not sweatshirts", level=Level.SHOULD),
                Postcondition("Provide a comprehensive list of available t-shirts", on="output", level=Level.MUST),
                Postcondition("Include key details like description, sizes, and available colors", on="output", level=Level.MUST),
                Postcondition("Provide photos of t-shirts", on="output", level=Level.SHOULD),
                Pathcondition("Ask customer for their preferences", level=Level.MUST),
                Pathcondition("Clarify if customer want t-shirts or sweatshirts", level=Level.SHOULD),
                Pathcondition("Do NOT recommend non-shirt products", level=Level.MUST),
                Pathcondition("Do NOT make up product features or specifications", level=Level.MUST),
            ],
        )
    ],
)

if __name__ == "__main__":
    customer_support_specs = Specifications(scenarios=[
        refund_eligible, 
        refund_ineligible, 
        shipping_status, 
        product_recommendation
    ])
    customer_support_specs.save("customer-support-specs.json")

Run Simulation

Conversations are simulated using the Relari Testbot for each scenario. Check out more details in the Simulation section.

Verification Results

The following is the verification results post-execution. We can see that the con-shipping-1 and con-product-1 contract are satisfied while the con-refund-1 and con-refund-2 contracts are not.

The detailed verification results with reasoning are saved in the output/verify_4cf4b3f5.json file.

────────────────────────────────────────────────────── Trace f3ec911405e760de9d9c64664c26ed6b ────────────────────────────────────────────────────── con-shipping-1 (SATISFIED)
┏━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ Type ┃ Qualifier ┃ Requirement ┃ Satisfied ┃ ┡━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩ │ PRE │ MUST │ Customer has provided order number order_01JGMMMPHCQM01FBAHY23FDA1 │ Yes │ │ PRE │ SHOULD │ Customer has provided email address for verification │ Yes │ │ PATH │ MUST │ Use email address to retrieve associated orders, and verify the order number doesn’t exist │ Yes │ │ PATH │ MUST │ Do NOT make up any shipping status, delivery dates, or any reason for the order not existing │ Yes │ │ PATH │ SHOULD │ Do NOT ask for unnecessary personal information beyond order number and email │ Yes │ │ POST │ MUST │ Inform customer that the order number does not exist in the system │ Yes │ │ POST │ SHOULD │ Provides escalation contact information to customer │ No │ └──────┴───────────┴──────────────────────────────────────────────────────────────────────────────────────────────┴───────────┘ ────────────────────────────────────────────────────── Trace 73b754a05920cc9ce5d26abbb73fe0d4 ────────────────────────────────────────────────────── con-refund-2_1 (UNSATISFIED)
┏━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ Type ┃ Qualifier ┃ Requirement ┃ Satisfied ┃ ┡━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩ │ PRE │ MUST │ Customer provides or selects order number order_01JGQF1YAYNFDV874MKFR33CAA │ Yes │ │ PRE │ SHOULD │ Customer provides email address if asked │ Yes │ │ PATH │ MUST │ Collect when the damage was first noticed, location of damage, and whether the item was worn │ Yes │ │ PATH │ MUST │ Check against the return policy and determine that the order is not eligible for a refund │ Yes │ │ POST │ MUST │ Deny the refund request │ No │ │ POST │ SHOULD │ Escalate to a manager if the customer insists on a refund │ No │ └──────┴───────────┴──────────────────────────────────────────────────────────────────────────────────────────────┴───────────┘ ────────────────────────────────────────────────────── Trace ed602aa95bbb251eb48e8f8ff56c8980 ────────────────────────────────────────────────────── con-refund-1 (UNSATISFIED)
┏━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ Type ┃ Qualifier ┃ Requirement ┃ Satisfied ┃ ┡━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩ │ PRE │ MUST │ Customer provides order number │ Yes │ │ PRE │ SHOULD │ Customer provides email address if asked │ No │ │ PRE │ SHOULD │ Customer should provided photo of damage to the sweater if asked │ No │ │ PATH │ MUST │ Collect when the damage was first noticed, location of damage, and the photo of damage │ No │ │ PATH │ MUST │ Check against the return policy and determine that the order is eligible for a refund │ Yes │ │ POST │ MUST │ Provide the refund id to the customer │ Yes │ │ POST │ MUST │ Process refund request as a store credit │ No │ └──────┴───────────┴────────────────────────────────────────────────────────────────────────────────────────┴───────────┘ ────────────────────────────────────────────────────── Trace 65d7b69d78be0a80638dd1304c20f0fa ────────────────────────────────────────────────────── con-product-1 (SATISFIED)
┏━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ Type ┃ Qualifier ┃ Requirement ┃ Satisfied ┃ ┡━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩ │ PRE │ MUST │ Customer express interest in shirts first (not t-shirts specifically) │ Yes │ │ PRE │ SHOULD │ Customer specifies that they want t-shirts and not sweatshirts │ Yes │ │ PATH │ MUST │ Ask customer for their preferences │ Yes │ │ PATH │ SHOULD │ Clarify if customer want t-shirts or sweatshirts │ No │ │ PATH │ MUST │ Do NOT recommend non-shirt products │ Yes │ │ PATH │ MUST │ Do NOT make up product features or specifications │ Yes │ │ POST │ MUST │ Provide a comprehensive list of available t-shirts │ Yes │ │ POST │ MUST │ Include key details like description, sizes, and available colors │ Yes │ │ POST │ SHOULD │ Provide photos of t-shirts │ Yes │ └──────┴───────────┴───────────────────────────────────────────────────────────────────────┴───────────┘

Example Certificate at Runtime

The following are two example runtime certificate received during the execution.

certificate.json
{
  "trace_id": "f3ec911405e760de9d9c64664c26ed6b",
  "specifications_id": "k5ssb3ge",
  "scenario_id": "sce-shipping-1",
  "contracts": {
    "con-shipping-1_1": {
      "status": "CERTIFIED",
      "info": {
        "req-qbwk88dr": {
          "satisfied": true,
          "explanation": "The trace confirms that the customer provided the order number 'order_01JGMMMPHCQM01FBAHY23FDA1' in the system input, fulfilling the requirement of order identification."
        },
        "req-wxly9mf4": {
          "satisfied": true,
          "explanation": "The trace shows that the customer supplied their email address (jane.doe@example.com) as required, enabling the system to verify their identity and retrieve order details."
        },
        "req-zjo1dfnv": {
          "satisfied": true,
          "explanation": "The trace confirms that the system attempted to retrieve the order details using the provided order number and email address. Since the order was not found, the system verified and confirmed that the order does not exist in the database, satisfying the requirement."
        },
        "req-he899akj": {
          "satisfied": true,
          "explanation": "The system accurately reported that the order number does not exist without fabricating shipping details, estimated delivery dates, or misleading information, thereby fulfilling the requirement."
        },
        "req-8yqyw5ea": {
          "satisfied": true,
          "explanation": "The trace demonstrates that only the necessary customer information (order number and email) was requested. No extraneous personal details were solicited, ensuring compliance with data privacy requirements."
        },
        "req-h8oumlrk": {
          "satisfied": true,
          "explanation": "The system explicitly stated that the order number provided does not exist in the database. This clear communication ensures that the customer is correctly informed about the nonexistence of the order, fulfilling the requirement."
        },
        "req-7p17btif": {
          "satisfied": true,
          "explanation": "The trace shows that the system provided customer support escalation details, including a phone number and an email address, ensuring that the user has the necessary information to seek further assistance."
        }
      }
    }
  }
}
certificate.json
{
  "trace_id": "73b754a05920cc9ce5d26abbb73fe0d4",
  "specifications_id": "k5ssb3ge",
  "scenario_id": "sce-refund-2_1",
  "contracts": {
    "con-52e81d70": {
      "status": "RUNNING",
      "info": {
        "req-u74uqpca": {
          "satisfied": true,
          "explanation": "The trace confirms that the customer provided a valid order number for reference, meeting the requirement for initiating a refund request."
        },
        "req-y3nwiq46": {
          "satisfied": true,
          "explanation": "The customer supplied their email address as requested, ensuring successful identity verification and compliance with refund processing requirements."
        },
        "req-2zw6dds6": {
          "satisfied": true,
          "explanation": "The trace confirms that the agent captured the location of the damage, when it was first noticed, and whether the item had been used, satisfying the requirement for collecting complete damage details."
        },
        "req-vgkoang1": {
          "satisfied": false,
          "explanation": "The requirement states that the order is not eligible for a refund, but the trace indicates that the system approved the refund despite the policy."
        },
        "req-09nf7klc": {
          "explanation": "Postcondition is only activated upon completion of the trace."
        },
        "req-2oj7gxko": {
          "explanation": "Postcondition is only activated upon completion of the trace."
        }
      }
    }
  }
}