Hallucination Scorer
A Vectara repack
Use
from transformers import pipeline
pairs = [
("The capital of France is Berlin.", "The capital of France is Paris."),
('I am in California', 'I am in United States.'),
('I am in United States', 'I am in California.'),
("A person on a horse jumps over a broken down airplane.", "A person is outdoors, on a horse."),
("A boy is jumping on skateboard in the middle of a red bridge.", "The boy skates down the sidewalk on a red bridge"),
("A man with blond-hair, and a brown shirt drinking out of a public water fountain.", "A blond man wearing a brown shirt is reading a book."),
("Mark Wahlberg was a fan of Manny.", "Manny was a fan of Mark Wahlberg.")
]
pipe = pipeline("pair-classification", model="tcapelle/hallu_scorer", trust_remote_code=True)
scores = pipe(pairs)
gt = [0.011061512865126133, 0.6473632454872131, 0.1290171593427658, 0.8969419002532959, 0.18462494015693665, 0.005031010136008263, 0.05432349815964699]
assert all(abs(s - g) < 1e-5 for s, g in zip(scores, gt))
