notebooks/batcheval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "pwsh"
    },
    "polyglot_notebook": {
     "kernelName": "pwsh"
    }
   },
   "outputs": [],
   "source": [
    "dotnet build ../examples/userstorygenerator/UserStoryGenerator\n",
    "dotnet build ../examples/batcheval/library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "#i \"nuget:https://api.nuget.org/v3/index.json\" \n",
    "#r \"nuget:Microsoft.SemanticKernel, 1.0.1\"\n",
    "#r \"nuget:OpenTelemetry.Exporter.Console, 1.7.0\"\n",
    "#r \"../examples/userstorygenerator/UserStoryGenerator/bin/Debug/net8.0/UserStoryGenerator.dll\"\n",
    "#r \"../examples/batcheval/library/bin/Debug/net8.0/BatchEvalCore.dll\"\n",
    "\n",
    "using Microsoft.DotNet.Interactive;\n",
    "using Microsoft.SemanticKernel;\n",
    "using UserStoryGenerator;\n",
    "using BatchEval.Core;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "var openAiEndpoint = await Microsoft.DotNet.Interactive.Kernel.GetInputAsync(\"Enter your OpenAI Endpoint: \");\n",
    "var openAiKey = await Microsoft.DotNet.Interactive.Kernel.GetInputAsync(\"Enter your OpenAI Key: \");\n",
    "var deploymentName = await Microsoft.DotNet.Interactive.Kernel.GetInputAsync(\"Enter your Deployment Name: \");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "var builder = Microsoft.SemanticKernel.Kernel.CreateBuilder();\n",
    "\n",
    "builder.AddAzureOpenAIChatCompletion(\n",
    "        deploymentName,\n",
    "        openAiEndpoint,\n",
    "        openAiKey);\n",
    "\n",
    "var kernel = builder.Build();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "internal struct UserInput\n",
    "{\n",
    "    public string Description { get; set; }\n",
    "    public string ProjectContext { get; set; }\n",
    "    public string Persona { get; set; }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "var dataFilePath = \"assets/data.jsonl\";\n",
    "var preview = await BatchEval.Core.JsonlUtils.Preview<UserInput>(dataFilePath);\n",
    "preview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "internal class UserStoryCreator : BatchEval.Core.IInputProcessor<UserInput>\n",
    "{\n",
    "    private readonly UserStorySkill userStoryGenerator;\n",
    "\n",
    "    public UserStoryCreator(Microsoft.SemanticKernel.Kernel kernel)\n",
    "    {\n",
    "        this.userStoryGenerator = UserStorySkill.Create(kernel);\n",
    "    }\n",
    "\n",
    "    public async Task<ModelOutput> Process(UserInput userInput)\n",
    "    {\n",
    "        var userStory = await userStoryGenerator.GetUserStory(\n",
    "            userInput.Description,\n",
    "            userInput.ProjectContext,\n",
    "            userInput.Persona);\n",
    "\n",
    "        return new ModelOutput() {\n",
    "            Input = $\"Generate a user story for {userInput.Persona} so it can {userInput.Description}\",\n",
    "            Output = $\"{userStory!.Title} - {userStory!.Description}\"\n",
    "        };\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    }
   },
   "outputs": [],
   "source": [
    "//var kernelFunctions = kernel.CreatePluginFromPromptDirectory(\"Prompts\");\n",
    "\n",
    "var batchEval = new BatchEval<UserInput>();\n",
    "\n",
    "batchEval\n",
    "    //.AddEvaluator(new PromptScoreEval(\"coherence\", kernel, kernelFunctions[\"coherence\"]))\n",
    "    .AddEvaluator(new LenghtEval());\n",
    "\n",
    "await batchEval\n",
    "    .WithInputProcessor(new UserStoryCreator(kernel))\n",
    "    .WithJsonl(dataFilePath)\n",
    "    .Run();"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".NET (C#)",
   "language": "C#",
   "name": ".net-csharp"
  },
  "language_info": {
   "name": "polyglot-notebook"
  },
  "polyglot_notebook": {
   "kernelInfo": {
    "defaultKernelName": "csharp",
    "items": [
     {
      "aliases": [],
      "name": "csharp"
     }
    ]
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}