generate_report.py

import os, sys, time
import wandb
import wandb_workspaces.reports.v2 as wr
wandb.require("core")

PROJECT_NAME="eval-llama-3.1-8b-fine-tune"
ENTITY="reviewco"

def main():

    api = wandb.Api()

    tstamp = sys.argv[1]

    i_check_run = 0
    i_check_count = 0
    i_kill = 40
    a_run_ids = ["production_" + tstamp, "candidate_" + tstamp]
    a_run_plus_states = ["production_" + tstamp + "-finished", "candidate_" + tstamp + "-finished"]
    while i_check_count < i_kill and i_check_run < len(a_run_plus_states):
        i_check_run = 0
        api.flush()
        runs = api.runs(path=f"{ENTITY}/{PROJECT_NAME}")
        for run in runs:
            s_run_plus_state = run.displayName + "-" + run.state
            # print("FOUND [" + s_run_plus_state + "]")
            if s_run_plus_state in a_run_plus_states:
                print("MATCHED [" + s_run_plus_state + "]")
                i_check_run = (i_check_run + 1)
                print("i_check_run [" + str(i_check_run) + "]")
        if i_check_run < len(a_run_plus_states):
            i_check_count = (i_check_count + 1)
            print("i_check_count [" + str(i_check_count) + "]")
            time.sleep(30)

    if i_check_run >= len(a_run_plus_states):
        generate_report(tstamp, a_run_ids)

def generate_report(tstamp, a_run_ids):
    report = wr.Report(
        entity=ENTITY,
        project=PROJECT_NAME,
        title="LLM fine-tuned model comparison report [" + tstamp + "]",
        description="Meta-Llama-3.1-8B Fine-Tuned with Alpaca",
        # width='fluid',
        width='fixed',
        blocks=[
            wr.TableOfContents(),
            wr.H1("Overview"),
            wr.P(
                text=[
                    "This report compares the ",
                    wr.InlineCode(text="candidate"),
                    " and ",
                    wr.InlineCode(text="production"),
                    " models for the ",
                    wr.InlineCode(text="Meta-Llama-3.1-8B"),
                    " fine-tuning task.",
                ]
            ),
            wr.H2("Github Information"),
            wr.P(
                [
                    "This report was automatically generated by ",
                    wr.Link("Github actions.", url="https://wandb.ai/")
                ]
            ),
            wr.H1("Evaluation Results"),
            wr.PanelGrid(
                runsets=[
                    wr.Runset(ENTITY, PROJECT_NAME, filters=f"displayName in {a_run_ids}")
                ],
                panels=[
                    wr.RunComparer(diff_only='split', layout={'w': 24, 'h': 9}),
                    wr.BarPlot(metrics=['hellaswag/acc'],orientation='v'),
                    wr.BarPlot(metrics=['lambada_openai/acc'],orientation='v'),
                    wr.BarPlot(metrics=['winogrande/acc'],orientation='v'),
                ]
            ),
            wr.H1("Deployment"),
            wr.P(
                text=[
                    "To run final tests and deploy this model, ",
                    "add the ",
                    wr.InlineCode(text="production"),
                    " alias to the ",
                    wr.InlineCode(text="candidate"),
                    " model. This will automatically run ",
                    wr.Link("this github action workflow", url="https://wandb.ai/"),
                    " to deploy ",
                    wr.InlineCode(text="Meta-Llama-3.1-8B:production"),
                    " to GCP ",
                    "(Provided you have the proper permissions to change the alias).",
                ]
            ),
            wr.H2("Model Artifact Information"),
            wr.P(
                text=[
                    "To access this model collection directly, ",
                    "please ",
                    wr.Link("click here", url="https://wandb.ai/registry/Llama%20Models?selectionPath=reviewco_RMVTAR2ST3HJ6%2Fwandb-registry-Llama+Models%2FLlama-3.1-8b&view=versions"),
                    ".",
                ]
            ),
            # https://wandb.ai/registry/Test%20Registry?selectionPath=russratshin-org%2Fwandb-registry-Test+Registry%2FLlama3+Fine-Tune&view=membership&tab=overview&version=latest
            # wr.QueryPanel(
            #     project("russratshin-org", "wandb-registry-Test Registry").artifact("Llama3 Fine-Tune")
            # ),
        ]
    )
    report.save()

def get_tstamp():
    # get timestamp
    from datetime import datetime
    tstamp = datetime.now().strftime("%Y%m%d%H%M%S%f") # 20240910113645272981
    # tstamp = tstamp[:16]
    return tstamp

if __name__ == "__main__":
    main()