xinmi/internal/application/service/metric/rouge_score.go

package metric

/*
# -*- coding: utf-8 -*-
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""ROUGE Metric Implementation

This is a very slightly version of:
https://github.com/pltrdy/seq2seq/blob/master/seq2seq/metrics/rouge.py

---

ROUGe metric implementation.

This is a modified and slightly extended verison of
https://github.com/miso-belica/sumy/blob/dev/sumy/evaluation/rouge.py.
*/

import (
	"strings"
)

type Ngrams struct {
	ngrams    map[string]int
	exclusive bool
}

func NewNgrams(exclusive bool) *Ngrams {
	return &Ngrams{ngrams: make(map[string]int), exclusive: exclusive}
}

func (n *Ngrams) Add(o string) {
	if n.exclusive {
		n.ngrams[o] = 1
	} else {
		n.ngrams[o]++
	}
}

func (n *Ngrams) Len() int {
	return len(n.ngrams)
}

func (n *Ngrams) Intersection(o *Ngrams) *Ngrams {
	intersection := NewNgrams(n.exclusive)
	for k := range n.ngrams {
		if _, ok := o.ngrams[k]; ok {
			intersection.Add(k)
		}
	}
	return intersection
}

func (n *Ngrams) BatchAdd(o []string) {
	for _, v := range o {
		n.Add(v)
	}
}

func (n *Ngrams) Union(others ...*Ngrams) *Ngrams {
	union := NewNgrams(n.exclusive)
	for k := range n.ngrams {
		union.Add(k)
	}
	for _, other := range others {
		for k := range other.ngrams {
			union.Add(k)
		}
	}
	return union
}

func getNgrams(n int, text []string, exclusive bool) *Ngrams {
	ngramSet := NewNgrams(exclusive)
	for i := 0; i <= len(text)-n; i++ {
		ngramSet.Add(strings.Join(text[i:i+n], " "))
	}
	return ngramSet
}

func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams {
	words := splitIntoWords(sentences)
	return getNgrams(n, words, exclusive)
}

func lcs(x, y []string) [][]int {
	n, m := len(x), len(y)
	table := make([][]int, n+1)
	for i := range table {
		table[i] = make([]int, m+1)
	}
	for i := 1; i <= n; i++ {
		for j := 1; j <= m; j++ {
			if x[i-1] == y[j-1] {
				table[i][j] = table[i-1][j-1] + 1
			} else {
				table[i][j] = max(table[i-1][j], table[i][j-1])
			}
		}
	}
	return table
}

func reconLcs(x, y []string, exclusive bool) *Ngrams {
	i, j := len(x), len(y)
	table := lcs(x, y)

	var reconFunc func(int, int) []string
	reconFunc = func(i, j int) []string {
		if i == 0 || j == 0 {
			return []string{}
		} else if x[i-1] == y[j-1] {
			return append(reconFunc(i-1, j-1), x[i-1])
		} else if table[i-1][j] > table[i][j-1] {
			return reconFunc(i-1, j)
		} else {
			return reconFunc(i, j-1)
		}
	}

	reconList := reconFunc(i, j)
	ngramList := NewNgrams(exclusive)
	for _, word := range reconList {
		ngramList.Add(word)
	}
	return ngramList
}

func rougeN(evaluatedSentences, referenceSentences []string, n int, rawResults, exclusive bool) map[string]float64 {
	evaluatedNgrams := getWordNgrams(n, evaluatedSentences, exclusive)
	referenceNgrams := getWordNgrams(n, referenceSentences, exclusive)
	referenceCount := referenceNgrams.Len()
	evaluatedCount := evaluatedNgrams.Len()

	overlappingNgrams := evaluatedNgrams.Intersection(referenceNgrams)
	overlappingCount := overlappingNgrams.Len()

	results := make(map[string]float64)
	if rawResults {
		results["hyp"] = float64(evaluatedCount)
		results["ref"] = float64(referenceCount)
		results["overlap"] = float64(overlappingCount)
		return results
	} else {
		return calculateRougeN(evaluatedCount, referenceCount, overlappingCount)
	}
}

func calculateRougeN(evaluatedCount, referenceCount, overlappingCount int) map[string]float64 {
	results := make(map[string]float64)
	if evaluatedCount == 0 {
		results["p"] = 0.0
	} else {
		results["p"] = float64(overlappingCount) / float64(evaluatedCount)
	}

	if referenceCount == 0 {
		results["r"] = 0.0
	} else {
		results["r"] = float64(overlappingCount) / float64(referenceCount)
	}

	results["f"] = 2.0 * ((results["p"] * results["r"]) / (results["p"] + results["r"] + 1e-8))
	return results
}

func unionLcs(evaluatedSentences []string, referenceSentence string, prevUnion *Ngrams, exclusive bool) (int, *Ngrams) {
	if prevUnion == nil {
		prevUnion = NewNgrams(exclusive)
	}

	lcsUnion := prevUnion
	prevCount := len(prevUnion.ngrams)
	referenceWords := splitIntoWords([]string{referenceSentence})

	combinedLcsLength := 0
	for _, evalS := range evaluatedSentences {
		evaluatedWords := splitIntoWords([]string{evalS})
		lcs := reconLcs(referenceWords, evaluatedWords, exclusive)
		combinedLcsLength += lcs.Len()
		lcsUnion = lcsUnion.Union(lcs)
	}

	newLcsCount := lcsUnion.Len() - prevCount
	return newLcsCount, lcsUnion
}

func rougeLSummaryLevel(
	evaluatedSentences, referenceSentences []string,
	rawResults, exclusive bool,
) map[string]float64 {
	referenceNgrams := NewNgrams(exclusive)
	referenceNgrams.BatchAdd(splitIntoWords(referenceSentences))
	m := referenceNgrams.Len()

	evaluatedNgrams := NewNgrams(exclusive)
	evaluatedNgrams.BatchAdd(splitIntoWords(evaluatedSentences))
	n := evaluatedNgrams.Len()

	unionLcsSumAcrossAllReferences := 0
	union := NewNgrams(exclusive)
	for _, refS := range referenceSentences {
		lcsCount, newUnion := unionLcs(evaluatedSentences, refS, union, exclusive)
		union = newUnion
		unionLcsSumAcrossAllReferences += lcsCount
	}

	llcs := unionLcsSumAcrossAllReferences

	var rLcs float64
	if m == 0 {
		rLcs = 0.0
	} else {
		rLcs = float64(llcs) / float64(m)
	}
	var pLcs float64
	if n == 0 {
		pLcs = 0.0
	} else {
		pLcs = float64(llcs) / float64(n)
	}

	fLcs := 2.0 * ((pLcs * rLcs) / (pLcs + rLcs + 1e-8))

	results := make(map[string]float64)
	if rawResults {
		results["hyp"] = float64(n)
		results["ref"] = float64(m)
		results["overlap"] = float64(llcs)
		return results
	} else {
		results["f"] = fLcs
		results["p"] = pLcs
		results["r"] = rLcs
		return results
	}
}