feat: add feedback management example and fix test script regression #62

README.md

@@ -58,6 +58,7 @@ This repository includes several examples demonstrating common use cases:
 - **[E2E Evaluation](./examples/e2e_eval)** - Run OpenAI experiments with automatic OpenTelemetry tracing linked to dataset examples
 - **[Record Experiment](./examples/record_experiment)** - Create datasets, examples, sessions, and batch ingest runs for experiments
 - **[Prompt Management](./examples/prompt_management)** - Create and manage prompt repositories, commits, and versions
+- **[Feedback Management](./examples/feedback)** - Create, update, list, and delete feedback for runs
 - **[OpenTelemetry Ingestion](./examples/otel_ingestion)** - Send OpenTelemetry traces to LangSmith with hierarchical span structure
 - **[OpenTelemetry + OpenAI](./examples/otel_openai)** - Make OpenAI API calls with manual OpenTelemetry tracing to LangSmith
 - **[OpenTelemetry + OpenAI (Go Client)](./examples/otel_go_client_openai)** - Automatic OpenTelemetry tracing for OpenAI API calls using sashabaranov/go-openai client wrapper

examples/feedback/README.md

@@ -0,0 +1,31 @@
+# Feedback Management Example
+
+This example demonstrates how to use the LangSmith Go SDK to manage feedback for your runs. Feedback is a crucial part of the LLM development lifecycle, allowing you to record scores, comments, and corrections from both human annotators and automated evaluators.
+
+## What this example covers
+
+1. **Creating a Run**: How to ingest a run into LangSmith so you have an entity to provide feedback on.
+2. **Submitting Feedback**: Using `client.Feedback.New` to submit scores, categorical values, and comments.
+3. **Updating Feedback**: Using `client.Feedback.Update` to modify existing feedback entries.
+4. **Listing Feedback**: Querying feedback for specific runs using `client.Feedback.List`.
+5. **Deleting Feedback**: Using `client.Feedback.Delete` to remove feedback entries.
+
+## Prerequisites
+
+- Go 1.22+
+- A LangSmith API Key (set as `LANGSMITH_API_KEY` environment variable)
+
+## Running the example
+
+```sh
+export LANGSMITH_API_KEY="your-api-key"
+go run ./examples/feedback
+```
+
+## Key Concepts
+
+- **Key**: The name of the feedback metric (e.g., "accuracy", "user_score", "helpfulness").
+- **Score**: A numerical value representing the feedback (typically between 0 and 1).
+- **Value**: A categorical or string value for the feedback (e.g., "correct", "incorrect").
+- **Comment**: An optional text description providing more context.
+- **Run ID**: The ID of the run the feedback is associated with.

examples/feedback/main.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/langchain-ai/langsmith-go"
+	"github.com/langchain-ai/langsmith-go/shared"
+)
+
+// Demonstrates how to manage feedback in LangSmith.
+//
+// This example shows:
+//   - Creating a run to provide feedback on
+//   - Submitting feedback (score, comment, correction) for a run
+//   - Updating existing feedback
+//   - Listing feedback with filters
+//   - Deleting feedback
+//
+// Prerequisites:
+//   - LANGSMITH_API_KEY: Your LangSmith API key
+//
+// Running:
+//
+//	go run ./examples/feedback
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	client := langsmith.NewClient()
+	ctx := context.Background()
+
+	printHeader()
+
+	// 1. Create a dummy run to provide feedback on
+	// In a real scenario, you would have a run ID from an actual LLM call.
+	runID := uuid.New().String()
+	fmt.Printf("1. Creating a dummy run (ID: %s)...\n", runID)
+
+	// We use IngestBatch to create a run quickly for this example
+	_, err := client.Runs.IngestBatch(ctx, langsmith.RunIngestBatchParams{
+		Post: langsmith.F([]langsmith.RunParam{
+			{
+				ID:        langsmith.F(runID),
+				Name:      langsmith.F("Feedback Example Run"),
+				RunType:   langsmith.F(langsmith.RunRunTypeLlm),
+				StartTime: langsmith.F(time.Now().Format(time.RFC3339)),
+				Inputs:    langsmith.F(map[string]interface{}{"question": "What is 2+2?"}),
+				Outputs:   langsmith.F(map[string]interface{}{"answer": "4"}),
+			},
+		}),
+	})
+	if err != nil {
+		return fmt.Errorf("creating dummy run: %w", err)
+	}
+	fmt.Println("   ✓ Run created")
+
+	// 2. Submit feedback for the run
+	fmt.Println("\n2. Submitting feedback for the run...")
+	feedback, err := client.Feedback.New(ctx, langsmith.FeedbackNewParams{
+		FeedbackCreateSchema: langsmith.FeedbackCreateSchemaParam{
+			RunID:   langsmith.F(runID),
+			Key:     langsmith.F("user_score"),
+			Score:   langsmith.F[langsmith.FeedbackCreateSchemaScoreUnionParam](shared.UnionFloat(1.0)),
+			Comment: langsmith.F("Excellent answer!"),
+			Value:   langsmith.F[langsmith.FeedbackCreateSchemaValueUnionParam](shared.UnionString("correct")),
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("submitting feedback: %w", err)
+	}
+	fmt.Printf("   ✓ Feedback submitted (ID: %s)\n", feedback.ID)
+
+	// 3. Update the feedback
+	fmt.Println("\n3. Updating the feedback comment...")
+	updatedFeedback, err := client.Feedback.Update(ctx, feedback.ID, langsmith.FeedbackUpdateParams{
+		Comment: langsmith.F("Excellent answer! Very concise."),
+	})
+	if err != nil {
+		return fmt.Errorf("updating feedback: %w", err)
+	}
+	fmt.Printf("   ✓ Feedback updated. New comment: %s\n", updatedFeedback.Comment)
+
+	// 4. List feedback for the run
+	fmt.Println("\n4. Listing feedback for the run...")
+	listRes, err := client.Feedback.List(ctx, langsmith.FeedbackListParams{
+		Run: langsmith.F[langsmith.FeedbackListParamsRunUnion](langsmith.FeedbackListParamsRunArray([]string{runID})),
+	})
+	if err != nil {
+		return fmt.Errorf("listing feedback: %w", err)
+	}
+	fmt.Printf("   ✓ Found %d feedback entries for run %s\n", len(listRes.Items), runID)
+	for _, f := range listRes.Items {
+		fmt.Printf("     - Key: %s, Score: %v, Comment: %s\n", f.Key, f.Score, f.Comment)
+	}
+
+	// 5. Delete the feedback
+	fmt.Println("\n5. Deleting the feedback...")
+	_, err = client.Feedback.Delete(ctx, feedback.ID)
+	if err != nil {
+		return fmt.Errorf("deleting feedback: %w", err)
+	}
+	fmt.Println("   ✓ Feedback deleted")
+
+	printSummary()
+	return nil
+}
+
+func printHeader() {
+	fmt.Println(strings.Repeat("=", 60))
+	fmt.Println("LangSmith Feedback Management Example")
+	fmt.Println(strings.Repeat("=", 60))
+	fmt.Println()
+}
+
+func printSummary() {
+	fmt.Println()
+	fmt.Println(strings.Repeat("=", 60))
+	fmt.Println("Example Complete!")
+	fmt.Println(strings.Repeat("=", 60))
+	fmt.Println("\nIn this example, we demonstrated how to:")
+	fmt.Println("  1. Create a run in LangSmith")
+	fmt.Println("  2. Submit programmatic feedback (scores and comments)")
+	fmt.Println("  3. Update existing feedback entries")
+	fmt.Println("  4. Query feedback by run ID")
+	fmt.Println("  5. Clean up by deleting feedback")
+	fmt.Println("\nFeedback is essential for improving LLM applications by:")
+	fmt.Println("  - Collecting human-in-the-loop ratings")
+	fmt.Println("  - Recording automated evaluator results")
+	fmt.Println("  - Tracking performance over time in your LangSmith projects")
+}

scripts/test

@@ -38,8 +38,12 @@ if ! is_overriding_api_base_url && ! prism_is_running ; then
   # When we exit this script, make sure to kill the background mock server process
   trap 'kill_server_on_port 4010' EXIT
 
-  # Start the dev server
-  ./scripts/mock --daemon
+  # Start the dev server if it exists
+  if [ -x "./scripts/mock" ]; then
+    ./scripts/mock --daemon
+  else
+    echo -e "${YELLOW}Warning:${NC} ./scripts/mock not found. Skipping background mock server start."
+  fi
 fi
 
 if is_overriding_api_base_url ; then