perf: speed up TryFrom<Scalar> for Primitive #45

Job	Run time
remove-bench-label	3s
sql-bench	0s
remove-sql-label	0s
bench / bench (compress-bench, Compression)	27m 59s
bench / bench (random-access-bench, Random Access, --features lance)	17m 40s
bench / sql / bench (tpch-s3-10, tpch, TPC-H SF=10 on S3, vortex-bench/data/tpch/10.0, s3://vortex-ci-benchmark...	22m 45s
bench / sql / bench (clickbench-nvme, clickbench, Clickbench on NVME, datafusion:parquet,datafusion:vortex,duck...	20m 42s
bench / sql / bench (fineweb-s3, fineweb, FineWeb S3, vortex-bench/data/fineweb, s3://vortex-ci-benchmark-datas...	15m 37s
bench / sql / bench (tpch-nvme, tpch, TPC-H SF=1 on NVME, datafusion:arrow,datafusion:parquet,datafusion:vortex...	9m 27s
bench / sql / bench (tpch-s3, tpch, TPC-H SF=1 on S3, vortex-bench/data/tpch/1.0, s3://vortex-ci-benchmark-data...	15m 14s
bench / sql / bench (statpopgen, statpopgen, Statistical and Population Genetics, duckdb:parquet,duckdb:vortex,...	18m 34s
bench / sql / bench (fineweb, fineweb, FineWeb NVMe, datafusion:parquet,datafusion:vortex,datafusion:vortex-com...	9m 42s
bench / sql / bench (polarsignals, polarsignals, PolarSignals Profiling, datafusion:vortex, 1)	9m 13s
bench / sql / bench (tpcds-nvme, tpcds, TPC-DS SF=1 on NVME, datafusion:parquet,datafusion:vortex,datafusion:vo...	12m 44s
bench / sql / bench (tpch-nvme-10, tpch, TPC-H SF=10 on NVME, datafusion:arrow,datafusion:parquet,datafusion:vo...	13m 45s
	3h 13m 25s

Provide feedback