Self-BLEU score calculation

[arandomg]

Hi!

Thanks for your great work. Could you please provide the details about how self-BLEU scores are calculated? I couldn't find them in the paper or the code. It will be super helpful!

[arandomg]

Hi, could you please provide the details? Thank you.

[arandomg]

Hi, checking in to see if there's an update

[yufeiwang63]

Here is an example script:

from typing import List, Callable, Union, Optional
import os
import random
import argparse
import glob
import numpy as np

import torch
import torch.nn.functional as F

from transformers import AutoTokenizer, AutoModel

import nltk
from fast_bleu import BLEU
from fast_bleu import SelfBLEU as FB_SelfBLEU

from sentence_transformers import SentenceTransformer


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--seed", type=int, default=42,
                        help="Random seed.")
    parser.add_argument("--device", type=str, default="cuda",
                        help="Torch device. Used in embedding cossim.")
    parser.add_argument("--tokenizer", type=str, default="gpt2",
                        help="Tokenizer for computing language-related metrics.")
    parser.add_argument("--metric", type=str, default="self_bleu",
                        help="Metric to be computed: self_bleu | embedding_cossim")
    ## Self BLEU
    parser.add_argument("--self-bleu-grams", type=int, nargs="+", default=[2, 3, 4, 5],
                        help="N-gram. Will average across all grams.")
    parser.add_argument("--self-bleu-sample-size", default=0.1,
                        help="Sample size when computing BLEU. -1 means the entire set. " + \
                             "Float to specificy the ratio of the entire set. Otherwise should be an integer.")
    parser.add_argument("--self-bleu-impl", type=str, default="in_house", choices=["in_house", "fast_bleu"],
                        help="Implementation of self BLEU.")
 
    args = parser.parse_args()
    
    if isinstance(args.self_bleu_sample_size, float):
        pass
    elif args.self_bleu_sample_size.isdigit():
        args.self_bleu_sample_size = int(args.self_bleu_sample_size)
    else:
        args.self_bleu_sample_size = float(args.self_bleu_sample_size)
  
    
    if args.seed > 0:
        random.seed(args.seed)
        np.random.seed(args.seed)
    
    return args


def main(args):
 
    
    # Set up tokenizer
    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)

    if args.metric == "self_bleu":
        self_bleu_config = dict(
            grams=args.self_bleu_grams,
            sample_size=args.self_bleu_sample_size,
            impl=args.self_bleu_impl,
        )
        self_bleu_scores = dict()
        
        data = [] #### your list of strings, e.g., task names, or task descriptions
        self_bleu_scores = compute_self_bleu(self_bleu_config, tokenizer, data)
        
        print("=== Self BLEU ===")
        print("Config")
        for k, v in self_bleu_config.items():
            print(f"  - {k}: {v}")
        avg_score = self_bleu_scores[target].mean()
        print(f"{target}: {avg_score:.8f}")
        print("=================")


def compute_self_bleu(config, tokenizer, data):
    self_bleu = SelfBleu(
        **config,
        tokenizer=lambda x: tokenizer.batch_decode(tokenizer(x, return_tensors="pt")["input_ids"][0].unsqueeze(1)),
    )
    self_bleu.append_reference(data)
    scores = self_bleu(data)
    
    return scores


class SelfBleu(object):
    def __init__(
        self, 
        grams: List[int] = [3, 4, 6],
        sample_size: Union[int, float] = -1,
        tokenizer: Callable = nltk.word_tokenize,
        impl: Optional[str] = "in_house",
    ) -> None:
        self.references = []
        self.grams = grams
        self.sample_size = sample_size
        self.tokenizer = tokenizer
        self.impl = impl

    def append_reference(
        self,
        ref: Union[str, List[str]],
    ) -> None:
        if isinstance(ref, list):
            self.references += list(map(self.tokenizer, ref))
        else:
            self.references.append(self.tokenizer(ref))

    def __call__(
        self,
        hypotheses: List[str],
    ) -> float:
        weights = {f"{n}-gram": ([1. / n] * n) for n in self.grams}
        tokenized_hypotheses = list(map(self.tokenizer, hypotheses))
        if self.impl == "in_house":
            if self.sample_size > 0:
                if isinstance(self.sample_size, float):
                    sample_size = int(self.sample_size * len(self.references))
                else:
                    sample_size = min(len(self.references), self.sample_size)
                bleu = BLEU(random.sample(self.references, k=sample_size), weights)
            else:
                bleu = BLEU(self.references, weights)
            scores = list(bleu.get_score(tokenized_hypotheses).values())
        elif self.impl == "fast_bleu":
            self_bleu = FB_SelfBLEU(tokenized_hypotheses, weights)
            scores = self_bleu.get_score()
            scores = np.array(list(scores.values()))
        else:
            raise NotImplementedError(f"Unrecognized implementation of SelfBLEU {self.impl}")
        
        return np.asarray(scores).mean(axis=0)

if __name__ == "__main__":
    args = parse_args()
    main(args)
   

[arandomg]

Hi @yufeiwang63

Thanks so much for making it available!! Unfortunately, I couldn't run the code due to the following errors:

RuntimeError: ffi_prep_cif_var failed at line bleu = BLEU(random.sample(self.references, k=sample_size), weights)

Also, the target variable seems not to be defined

        avg_score = self_bleu_scores[target].mean()
        print(f"{target}: {avg_score:.8f}")

Would you mind spending several minutes more to check these? Thanks again!