Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AI Test Tool] Add Multi-Turn Accuracy & Export #2419

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Code review comments
stkillen committed Jan 27, 2025
commit 2e82858adc4471e52ae20a286cbcef88a3326c2a
20 changes: 10 additions & 10 deletions src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al
Original file line number Diff line number Diff line change
@@ -18,12 +18,12 @@ codeunit 149043 "AIT Test Context Impl."
var
AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
GlobalTestOutputJson: Codeunit "Test Output Json";
Accuracy: Decimal;
GlobalAccuracy: Decimal;
CurrentTurn: Integer;
NumberOfTurns: Integer;
IsMultiTurn: Boolean;
AccuracySetManually: Boolean;
AccuracyErr: Label 'Accuracy must be between 0.0 and 1.0.';
AccuracyErr: Label 'Accuracy must be between 0 and 1.';
AnswerTok: Label 'answer', Locked = true;
ContextTok: Label 'context', Locked = true;
GroundTruthTok: Label 'ground_truth', Locked = true;
@@ -152,25 +152,25 @@ codeunit 149043 "AIT Test Context Impl."
/// <summary>
/// Sets the accuracy of the test.
/// </summary>
/// <param name="AccuracyPct">The accuracy as a decimal between 0 and 1.</param>
procedure SetAccuracy(AccuracyPct: Decimal)
/// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
procedure SetAccuracy(Accuracy: Decimal)
begin
if (AccuracyPct < 0) or (AccuracyPct > 1.0) then
if (Accuracy < 0) or (Accuracy > 1) then
Error(AccuracyErr);

AccuracySetManually := true;
Accuracy := AccuracyPct;
GlobalAccuracy := Accuracy;
end;

/// <summary>
/// Gets the accuracy of the test. Can only be retrieved if the accuracy of the test was already set manually.
/// </summary>
/// <param name="AccuracyPct">The accuracy as a decimal between 0 and 1.</param>
/// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
/// <returns>True if it was possible to get the accuracy, false otherwise.</returns>
procedure GetAccuracy(var AccuracyPct: Decimal): Boolean
procedure GetAccuracy(var Accuracy: Decimal): Boolean
begin
if AccuracySetManually then begin
AccuracyPct := Accuracy;
Accuracy := GlobalAccuracy;
exit(true);
end;

@@ -246,7 +246,7 @@ codeunit 149043 "AIT Test Context Impl."
TurnsInputJson: Codeunit "Test Input Json";
begin
AccuracySetManually := false;
Accuracy := 0;
GlobalAccuracy := 0;
CurrentTurn := 1;
GlobalTestOutputJson.Initialize();
TurnsInputJson := TestInput.GetTestInput().ElementExists(TurnsTok, IsMultiTurn);
Original file line number Diff line number Diff line change
@@ -168,7 +168,7 @@ codeunit 149042 "AIT Test Run Iteration"
GlobalNumberOfTurnsForLastTestMethodLine := 1;

// Update Test Accuracy
GlobalTestAccuracy := 0.0;
GlobalTestAccuracy := 0;

GlobalSessionAITokenUsed := AOAIToken.GetTotalServerSessionTokensConsumed();

6 changes: 3 additions & 3 deletions src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al
Original file line number Diff line number Diff line change
@@ -68,14 +68,14 @@ page 149033 "AIT Log Entries"
{
StyleExpr = StatusStyleExpr;
}
field(Accuracy; Rec.Accuracy)
field(Accuracy; Rec."Test Method Line Accuracy")
{
}
field("No. of Turns Passed"; Rec."No. of Turns Passed")
{
Visible = false;
}
field("No. of Turns Executed"; Rec."No. of Turns Executed")
field("No. of Turns"; Rec."No. of Turns")
{
Visible = false;
}
@@ -357,7 +357,7 @@ page 149033 "AIT Log Entries"
local procedure SetTurnsStyleExpr()
begin
case Rec."No. of Turns Passed" of
Rec."No. of Turns Executed":
Rec."No. of Turns":
TurnsStyleExpr := 'Favorable';
0:
TurnsStyleExpr := 'Unfavorable';
10 changes: 5 additions & 5 deletions src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al
Original file line number Diff line number Diff line change
@@ -159,20 +159,20 @@ table 149034 "AIT Log Entry"
{
Caption = 'Output Data';
}
field(40; "No. of Turns Executed"; Integer)
field(40; "No. of Turns"; Integer)
{
Caption = 'Total number of turns';
ToolTip = 'Specifies the total number of turns executed.';
ToolTip = 'Specifies the total number of turns.';
}
field(41; "No. of Turns Passed"; Integer)
{
Caption = 'Number of turns passed';
ToolTip = 'Specifies the number of turns passed.';
}
field(45; Accuracy; Decimal)
field(45; "Test Method Line Accuracy"; Decimal)
{
Caption = 'Accuracy';
ToolTip = 'Specifies the accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually by the test.';
Caption = 'Test Method Line Accuracy';
ToolTip = 'Specifies the accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually in the test.';
}
field(50; "Tokens Consumed"; Integer)
{
2 changes: 1 addition & 1 deletion src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al
Original file line number Diff line number Diff line change
@@ -121,7 +121,7 @@ page 149032 "AIT Run History"
AITLogEntryCodeunit.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No. Filter", Rec.Version);
end;
}
field("Accuracy - By Version"; Rec.Accuracy)
field("Accuracy - By Version"; Rec."Accuracy Per Version")
{
Visible = ViewBy = ViewBy::Version;
Caption = 'Accuracy';
6 changes: 3 additions & 3 deletions src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al
Original file line number Diff line number Diff line change
@@ -73,13 +73,13 @@ table 149036 "AIT Run History"
FieldClass = FlowField;
CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), Version = field("Version"), "Test Method Line No." = field("Line No. Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(14; "Accuracy"; Decimal)
field(14; "Accuracy Per Version"; Decimal)
{
Caption = 'Accuracy';
ToolTip = 'Specifies the average accuracy of the version.';
Editable = false;
FieldClass = FlowField;
CalcFormula = average("AIT Log Entry"."Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(20; "No. of Tests Executed - By Tag"; Integer)
{
@@ -119,7 +119,7 @@ table 149036 "AIT Run History"
ToolTip = 'Specifies the average accuracy of the tag.';
Editable = false;
FieldClass = FlowField;
CalcFormula = average("AIT Log Entry"."Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
}

4 changes: 2 additions & 2 deletions src/Tools/AI Test Toolkit/src/Logs/AITTestSummary.Report.al
Original file line number Diff line number Diff line change
@@ -32,10 +32,10 @@ report 149030 "AIT Test Summary"
column(Status; Results.Status)
{
}
column(Accuracy; Results.Accuracy)
column(Accuracy; Results."Test Method Line Accuracy")
{
}
column(TurnsExecuted; Results."No. of Turns Executed")
column(TurnsExecuted; Results."No. of Turns")
{
}
column(TurnsPassed; Results."No. of Turns Passed")
Original file line number Diff line number Diff line change
@@ -166,13 +166,13 @@ table 149032 "AIT Test Method Line"
FieldClass = FlowField;
CalcFormula = count("AIT Log Entry" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Base Version Filter")));
}
field(40; "No. of Turns Executed"; Integer)
field(40; "No. of Turns"; Integer)
{
Caption = 'No. of Turns Executed';
ToolTip = 'Specifies the total number of turns for the test line.';
Editable = false;
FieldClass = FlowField;
CalcFormula = sum("AIT Log Entry"."No. of Turns Executed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
CalcFormula = sum("AIT Log Entry"."No. of Turns" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(41; "No. of Turns Passed"; Integer)
{
@@ -182,13 +182,13 @@ table 149032 "AIT Test Method Line"
FieldClass = FlowField;
CalcFormula = sum("AIT Log Entry"."No. of Turns Passed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(45; Accuracy; Decimal)
field(45; "Test Method Line Accuracy"; Decimal)
{
Caption = 'Accuracy';
ToolTip = 'Specifies the average accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually by the test.';
Editable = false;
FieldClass = FlowField;
CalcFormula = average("AIT Log Entry"."Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(101; "AL Test Suite"; Code[10])
{
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ page 149034 "AIT Test Method Lines"
AITLogEntry.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No.", AITTestSuite.Version);
end;
}
field(Accuracy; Rec.Accuracy)
field(Accuracy; Rec."Test Method Line Accuracy")
{
}
field(TurnsText; TurnsText)
@@ -93,15 +93,15 @@ page 149034 "AIT Test Method Lines"
AITLogEntry.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No.", AITTestSuite."Base Version");
end;
}
field("No. of Turns"; Rec."No. of Turns Executed")
field("No. of Turns"; Rec."No. of Turns")
{
Visible = false;
}
field("No. of Turns Passed"; Rec."No. of Turns Passed")
{
Visible = false;
}
field("No. of Turns Failed"; Rec."No. of Turns Executed" - Rec."No. of Turns Passed")
field("No. of Turns Failed"; Rec."No. of Turns" - Rec."No. of Turns Passed")
{
Visible = false;
Editable = false;
Original file line number Diff line number Diff line change
@@ -167,7 +167,7 @@ table 149030 "AIT Test Suite"
ToolTip = 'Specifies the average accuracy of the test suite. The accuracy is calculated as the percentage of turns that passed or can be set manually by the test.';
Editable = false;
FieldClass = FlowField;
CalcFormula = average("AIT Log Entry"."Accuracy" where("Test Suite Code" = field("Code"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Code"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
}
field(50; "Test Runner Id"; Integer)
{
Original file line number Diff line number Diff line change
@@ -359,9 +359,9 @@ codeunit 149034 "AIT Test Suite Mgt."

AITLogEntry."Procedure Name" := CurrentTestMethodLine.Function;
AITLogEntry."Tokens Consumed" := AITTestRunIteration.GetAITokenUsedByLastTestMethodLine();
AITLogEntry."No. of Turns Executed" := AITTestRunIteration.GetNumberOfTurnsForLastTestMethodLine();
AITLogEntry."No. of Turns" := AITTestRunIteration.GetNumberOfTurnsForLastTestMethodLine();
AITLogEntry."No. of Turns Passed" := AITTestRunIteration.GetNumberOfTurnsPassedForLastTestMethodLine();
AITLogEntry.Accuracy := AITTestRunIteration.GetAccuracyForLastTestMethodLine();
AITLogEntry."Test Method Line Accuracy" := AITTestRunIteration.GetAccuracyForLastTestMethodLine();
AITLogEntry.Insert(true);

Commit();
@@ -380,13 +380,13 @@ codeunit 149034 "AIT Test Suite Mgt."

internal procedure GetTurnsAsText(var AITTestMethodLine: Record "AIT Test Method Line"): Text
begin
AITTestMethodLine.CalcFields("No. of Turns Passed", "No. of Turns Executed");
exit(StrSubstNo(TurnsLbl, AITTestMethodLine."No. of Turns Passed", AITTestMethodLine."No. of Turns Executed"));
AITTestMethodLine.CalcFields("No. of Turns Passed", "No. of Turns");
exit(StrSubstNo(TurnsLbl, AITTestMethodLine."No. of Turns Passed", AITTestMethodLine."No. of Turns"));
end;

internal procedure GetTurnsAsText(var AITLogEntry: Record "AIT Log Entry"): Text
begin
exit(StrSubstNo(TurnsLbl, AITLogEntry."No. of Turns Passed", AITLogEntry."No. of Turns Executed"));
exit(StrSubstNo(TurnsLbl, AITLogEntry."No. of Turns Passed", AITLogEntry."No. of Turns"));
end;

internal procedure GetAvgDuration(AITTestMethodLine: Record "AIT Test Method Line"): Integer