model	family	comparison	n_unique_targets	mean_paired_difference	bootstrap_ci95_low	bootstrap_ci95_high	alternative	paired_sign_flip_p	bootstrap_replicates	sign_flip_permutations	holm_p_within_model_family
negative_feature_row_shuffle	rmse	rmse_vs_zero	2160	-0.0056299424057619	-0.0066609216486405	-0.0046168482833309	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_feature_row_shuffle	rmse	rmse_vs_train_mean	2160	0.0002036708545077	0.0001183694204815	0.0002910665913347	less	1.0	10000	50000	1.0
negative_feature_row_shuffle	directional	raw_cosine_vs_train_mean	2160	-0.0017637540500665	-0.0025815557849972	-0.0010660825699645	greater	1.0	10000	50000	1.0
negative_feature_row_shuffle	directional	perturbation_specific_residual_cosine_vs_zero	2160	-0.0062129220766817	-0.0182190275523953	0.0058341433768523	greater	0.8454030919381612	10000	50000	1.0
negative_outcome_coordinate_permutation	rmse	rmse_vs_zero	2160	0.0156305948168867	0.0153298130997723	0.0159375462114931	less	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	rmse	rmse_vs_train_mean	2160	0.0214642080771564	0.0205865762759155	0.0223422956949151	less	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	directional	raw_cosine_vs_train_mean	2160	-0.3506010896074565	-0.3644286725671288	-0.3367248739853999	greater	1.0	10000	50000	1.0
negative_outcome_coordinate_permutation	directional	perturbation_specific_residual_cosine_vs_zero	2160	0.0987154842525689	0.0843284419009648	0.1134096635190066	greater	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_response_row_shuffle	rmse	rmse_vs_zero	2160	-0.0056192696560174	-0.0066603477506173	-0.0045957438380638	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
negative_response_row_shuffle	rmse	rmse_vs_train_mean	2160	0.0002143436042522	0.0001043743794973	0.000327754271415	less	1.0	10000	50000	1.0
negative_response_row_shuffle	directional	raw_cosine_vs_train_mean	2160	-0.0016405812797765	-0.0024447970441554	-0.0009086892247847	greater	0.999980000399992	10000	50000	1.0
negative_response_row_shuffle	directional	perturbation_specific_residual_cosine_vs_zero	2160	-0.0069696009078496	-0.0191503613497614	0.0052014838235112	greater	0.872122557548849	10000	50000	1.0
safe_external_v1	rmse	rmse_vs_zero	2160	-0.0063136818249606	-0.0073800105797083	-0.0052668378806535	less	1.999960000799984e-05	10000	50000	3.999920001599968e-05
safe_external_v1	rmse	rmse_vs_train_mean	2160	-0.0004800685646909	-0.0007113322425909	-0.0002534822155117	less	5.999880002399952e-05	10000	50000	5.999880002399952e-05
safe_external_v1	directional	raw_cosine_vs_train_mean	2160	0.0016365594825738	0.0003947280861909	0.0028060510205708	greater	0.0035599288014239	10000	50000	0.0035599288014239
safe_external_v1	directional	perturbation_specific_residual_cosine_vs_zero	2160	0.0454424261507449	0.0295758587363511	0.061217912452975	greater	1.999960000799984e-05	10000	50000	3.999920001599968e-05