model	family	comparison	n_unique_targets	mean_paired_difference	bootstrap_ci95_low	bootstrap_ci95_high	alternative	paired_sign_flip_p	bootstrap_replicates	sign_flip_permutations	holm_p_within_model_family
negative_feature_row_shuffle	rmse	rmse_vs_zero	2160	-0.005495467055933895	-0.006543208480234844	-0.004457924364833162	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_feature_row_shuffle	rmse	rmse_vs_train_mean	2160	0.0003224892455532595	0.00019610658917507088	0.000473900313394282	less	1.0	10000	50000	1.0
negative_feature_row_shuffle	directional	raw_cosine_vs_train_mean	2160	-0.0013378903398410735	-0.001971338126782956	-0.0007431447403994887	greater	1.0	10000	50000	1.0
negative_feature_row_shuffle	directional	perturbation_specific_residual_cosine_vs_zero	2160	-0.0012359058803566667	-0.01288690536330486	0.010242535564406586	greater	0.5854282914341713	10000	50000	1.0
negative_outcome_coordinate_permutation	rmse	rmse_vs_zero	2160	0.015504683381705372	0.01520699586766703	0.0158089969202626	less	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	rmse	rmse_vs_train_mean	2160	0.021322639683192528	0.020455790182006443	0.02219520496398521	less	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	directional	raw_cosine_vs_train_mean	2160	-0.3481725753432392	-0.36190170086722373	-0.3343882173407399	greater	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	directional	perturbation_specific_residual_cosine_vs_zero	2160	0.09901257126415051	0.08465268492716994	0.11356312785787681	greater	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_response_row_shuffle	rmse	rmse_vs_zero	2160	-0.005647392936602787	-0.006687207428083099	-0.004626710126542969	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_response_row_shuffle	rmse	rmse_vs_train_mean	2160	0.0001705633648843677	7.918638332436483e-05	0.0002676070964446774	less	0.99980000399992	10000	50000	0.99980000399992
negative_response_row_shuffle	directional	raw_cosine_vs_train_mean	2160	-0.001621348422998347	-0.0023824151126993656	-0.0009219379836334461	greater	1.0	10000	50000	1.0
negative_response_row_shuffle	directional	perturbation_specific_residual_cosine_vs_zero	2160	0.01086478937235774	-0.0002385166147339784	0.021878535232418905	greater	0.027179456410871782	10000	50000	0.054358912821743564
safe_external_v1	rmse	rmse_vs_zero	2160	-0.006295363780938917	-0.007353199810647682	-0.00525590590942927	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
safe_external_v1	rmse	rmse_vs_train_mean	2160	-0.0004774074794517623	-0.0006990205378840781	-0.0002535452632474955	less	3.999920001599968e-05	10000	50000	3.999920001599968e-05
safe_external_v1	directional	raw_cosine_vs_train_mean	2160	0.001006855543252001	-0.00034945167581503883	0.0023256907720466775	greater	0.06881862362752746	10000	50000	0.06881862362752746
safe_external_v1	directional	perturbation_specific_residual_cosine_vs_zero	2160	0.047808347938553195	0.03209513525216001	0.06335820132626765	greater	1.999960000799984e-05	10000	50000	3.999920001599968e-05