WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/application/service/metric/rouge_score.go

@@ -0,0 +1,250 @@
+package metric
+
+/*
+# -*- coding: utf-8 -*-
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ROUGE Metric Implementation
+
+This is a very slightly version of:
+https://github.com/pltrdy/seq2seq/blob/master/seq2seq/metrics/rouge.py
+
+---
+
+ROUGe metric implementation.
+
+This is a modified and slightly extended verison of
+https://github.com/miso-belica/sumy/blob/dev/sumy/evaluation/rouge.py.
+*/
+
+import (
+	"strings"
+)
+
+type Ngrams struct {
+	ngrams    map[string]int
+	exclusive bool
+}
+
+func NewNgrams(exclusive bool) *Ngrams {
+	return &Ngrams{ngrams: make(map[string]int), exclusive: exclusive}
+}
+
+func (n *Ngrams) Add(o string) {
+	if n.exclusive {
+		n.ngrams[o] = 1
+	} else {
+		n.ngrams[o]++
+	}
+}
+
+func (n *Ngrams) Len() int {
+	return len(n.ngrams)
+}
+
+func (n *Ngrams) Intersection(o *Ngrams) *Ngrams {
+	intersection := NewNgrams(n.exclusive)
+	for k := range n.ngrams {
+		if _, ok := o.ngrams[k]; ok {
+			intersection.Add(k)
+		}
+	}
+	return intersection
+}
+
+func (n *Ngrams) BatchAdd(o []string) {
+	for _, v := range o {
+		n.Add(v)
+	}
+}
+
+func (n *Ngrams) Union(others ...*Ngrams) *Ngrams {
+	union := NewNgrams(n.exclusive)
+	for k := range n.ngrams {
+		union.Add(k)
+	}
+	for _, other := range others {
+		for k := range other.ngrams {
+			union.Add(k)
+		}
+	}
+	return union
+}
+
+func getNgrams(n int, text []string, exclusive bool) *Ngrams {
+	ngramSet := NewNgrams(exclusive)
+	for i := 0; i <= len(text)-n; i++ {
+		ngramSet.Add(strings.Join(text[i:i+n], " "))
+	}
+	return ngramSet
+}
+
+func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams {
+	words := splitIntoWords(sentences)
+	return getNgrams(n, words, exclusive)
+}
+
+func lcs(x, y []string) [][]int {
+	n, m := len(x), len(y)
+	table := make([][]int, n+1)
+	for i := range table {
+		table[i] = make([]int, m+1)
+	}
+	for i := 1; i <= n; i++ {
+		for j := 1; j <= m; j++ {
+			if x[i-1] == y[j-1] {
+				table[i][j] = table[i-1][j-1] + 1
+			} else {
+				table[i][j] = max(table[i-1][j], table[i][j-1])
+			}
+		}
+	}
+	return table
+}
+
+func reconLcs(x, y []string, exclusive bool) *Ngrams {
+	i, j := len(x), len(y)
+	table := lcs(x, y)
+
+	var reconFunc func(int, int) []string
+	reconFunc = func(i, j int) []string {
+		if i == 0 || j == 0 {
+			return []string{}
+		} else if x[i-1] == y[j-1] {
+			return append(reconFunc(i-1, j-1), x[i-1])
+		} else if table[i-1][j] > table[i][j-1] {
+			return reconFunc(i-1, j)
+		} else {
+			return reconFunc(i, j-1)
+		}
+	}
+
+	reconList := reconFunc(i, j)
+	ngramList := NewNgrams(exclusive)
+	for _, word := range reconList {
+		ngramList.Add(word)
+	}
+	return ngramList
+}
+
+func rougeN(evaluatedSentences, referenceSentences []string, n int, rawResults, exclusive bool) map[string]float64 {
+	evaluatedNgrams := getWordNgrams(n, evaluatedSentences, exclusive)
+	referenceNgrams := getWordNgrams(n, referenceSentences, exclusive)
+	referenceCount := referenceNgrams.Len()
+	evaluatedCount := evaluatedNgrams.Len()
+
+	overlappingNgrams := evaluatedNgrams.Intersection(referenceNgrams)
+	overlappingCount := overlappingNgrams.Len()
+
+	results := make(map[string]float64)
+	if rawResults {
+		results["hyp"] = float64(evaluatedCount)
+		results["ref"] = float64(referenceCount)
+		results["overlap"] = float64(overlappingCount)
+		return results
+	} else {
+		return calculateRougeN(evaluatedCount, referenceCount, overlappingCount)
+	}
+}
+
+func calculateRougeN(evaluatedCount, referenceCount, overlappingCount int) map[string]float64 {
+	results := make(map[string]float64)
+	if evaluatedCount == 0 {
+		results["p"] = 0.0
+	} else {
+		results["p"] = float64(overlappingCount) / float64(evaluatedCount)
+	}
+
+	if referenceCount == 0 {
+		results["r"] = 0.0
+	} else {
+		results["r"] = float64(overlappingCount) / float64(referenceCount)
+	}
+
+	results["f"] = 2.0 * ((results["p"] * results["r"]) / (results["p"] + results["r"] + 1e-8))
+	return results
+}
+
+func unionLcs(evaluatedSentences []string, referenceSentence string, prevUnion *Ngrams, exclusive bool) (int, *Ngrams) {
+	if prevUnion == nil {
+		prevUnion = NewNgrams(exclusive)
+	}
+
+	lcsUnion := prevUnion
+	prevCount := len(prevUnion.ngrams)
+	referenceWords := splitIntoWords([]string{referenceSentence})
+
+	combinedLcsLength := 0
+	for _, evalS := range evaluatedSentences {
+		evaluatedWords := splitIntoWords([]string{evalS})
+		lcs := reconLcs(referenceWords, evaluatedWords, exclusive)
+		combinedLcsLength += lcs.Len()
+		lcsUnion = lcsUnion.Union(lcs)
+	}
+
+	newLcsCount := lcsUnion.Len() - prevCount
+	return newLcsCount, lcsUnion
+}
+
+func rougeLSummaryLevel(
+	evaluatedSentences, referenceSentences []string,
+	rawResults, exclusive bool,
+) map[string]float64 {
+	referenceNgrams := NewNgrams(exclusive)
+	referenceNgrams.BatchAdd(splitIntoWords(referenceSentences))
+	m := referenceNgrams.Len()
+
+	evaluatedNgrams := NewNgrams(exclusive)
+	evaluatedNgrams.BatchAdd(splitIntoWords(evaluatedSentences))
+	n := evaluatedNgrams.Len()
+
+	unionLcsSumAcrossAllReferences := 0
+	union := NewNgrams(exclusive)
+	for _, refS := range referenceSentences {
+		lcsCount, newUnion := unionLcs(evaluatedSentences, refS, union, exclusive)
+		union = newUnion
+		unionLcsSumAcrossAllReferences += lcsCount
+	}
+
+	llcs := unionLcsSumAcrossAllReferences
+
+	var rLcs float64
+	if m == 0 {
+		rLcs = 0.0
+	} else {
+		rLcs = float64(llcs) / float64(m)
+	}
+	var pLcs float64
+	if n == 0 {
+		pLcs = 0.0
+	} else {
+		pLcs = float64(llcs) / float64(n)
+	}
+
+	fLcs := 2.0 * ((pLcs * rLcs) / (pLcs + rLcs + 1e-8))
+
+	results := make(map[string]float64)
+	if rawResults {
+		results["hyp"] = float64(n)
+		results["ref"] = float64(m)
+		results["overlap"] = float64(llcs)
+		return results
+	} else {
+		results["f"] = fLcs
+		results["p"] = pLcs
+		results["r"] = rLcs
+		return results
+	}
+}