機械翻訳の精度評価と改善手法

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

import nltk

bleu_scores = []
for reference, pre in zip(reference_translations, prediction):
    reference_tokens = nltk.word_tokenize(reference.lower())
    pre_tokens = nltk.word_tokenize(pre.lower())

    if not reference_tokens or not pre_tokens:
        continue

    bleu_score = nltk.translate.bleu_score.sentence_bleu(
        [reference_tokens], pre_tokens,
        smoothing_function=nltk.translate.bleu_score.SmoothingFunction().method2)
    bleu_scores.append(bleu_score)

average_bleu_score = sum(bleu_scores) / len(bleu_scores)
print("Average BLEU score:", average_bleu_score)

1
2
3
4
5
6
7
8

from bleurt import score

checkpoint = "/path/to/BLEURT-20"
scorer = score.BleurtScorer(checkpoint)
scores = scorer.score(references=reference_translations, candidates=prediction)

total_score = sum(scores) / len(scores)
print("Total Score:", total_score)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

from comet import download_model, load_from_checkpoint

model_path = download_model("Unbabel/wmt22-comet-da")
model = load_from_checkpoint(model_path)

data = []
for src, pre, reference in zip(source_sentences, preds, reference_translations):
    data.append({
        "src": src,
        "mt": pre,
        "ref": reference
    })

model_output = model.predict(data, batch_size=8, gpus=0)
print(model_output)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

import requests, uuid, json

endpoint = ""
subscription_key = ""
location = ""

path = '/translate'
constructed_url = endpoint + path

params = {
    'api-version': '3.0',
    'from': 'zh',
    'to': 'en'
}

headers = {
    'Ocp-Apim-Subscription-Key': subscription_key,
    'Ocp-Apim-Subscription-Region': location,
    'Content-type': 'application/json',
    'X-ClientTraceId': str(uuid.uuid4())
}

body = []
for i in source_sentences:
    body.append({'text': i})

request = requests.post(constructed_url, params=params, headers=headers, json=body)
response = request.json()

1
2
3
4
5
6
7
8

import deepl

API_KEY = 'your-api-key'
source_lang = 'ZH'
target_lang = 'EN-US'

translator = deepl.Translator(API_KEY)
results = translator.translate_text(source_sentences, source_lang=source_lang, target_lang=target_lang)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

import requests

def translate_texts(texts, target_language):
    api_key = 'your-api-key'
    url = 'https://translation.googleapis.com/language/translate/v2'
    translations = []

    for text in texts:
        params = {
            'key': api_key,
            'q': text,
            'target': target_language
        }
        response = requests.get(url, params=params)
        if response.status_code == 200:
            data = response.json()
            translated_text = data['data']['translations'][0]['translatedText']
            translations.append(translated_text)

    return translations

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

import openai

def translate_text(text_list):
    openai.api_key = 'your-api-key'
    translations = []

    for text in text_list:
        messages = [
            {"role": "system", "content": "You are a translation assistant from Chinese to English. Some rules to remember:\n\n- Do not add extra blank lines.\n- It is important to maintain the accuracy of the contents, but we don't want the output to read like it's been translated. So instead of translating word by word, prioritize naturalness and ease of communication."},
            {"role": "user", "content": text}
        ]

        response = openai.ChatCompletion.create(
            model='gpt-4',
            messages=messages,
            max_tokens=100,
            temperature=0.7,
            timeout=30
        )

        choices = response['choices']
        if len(choices) > 0:
            translation = choices[0]['message']['content']
            translations.append(translation)

    return translations

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

import requests, uuid, json
import openai
from comet import download_model, load_from_checkpoint

def translate_with_fallback(text):
    translation_from_Azure = Azure_translation(text)
    model_path = download_model("Unbabel/wmt22-comet-da")
    model = load_from_checkpoint(model_path)
    refined_translation = []
    indices_to_correct = []

    for i in range(len(translation_from_Azure)):
        data = [{
            "src": source_sentences[i],
            "mt": translation_from_Azure[i],
            "ref": reference_translations[i]
        }]
        res = model.predict(data, batch_size=1, gpus=0)
        if res.scores[0] < 0.81:
            indices_to_correct.append(i)

    sentences_to_correct = [source_sentences[i] for i in indices_to_correct]
    corrected_sentences = gpt_translation(sentences_to_correct)

    corrected_index = 0
    for i in range(len(translation_from_Azure)):
        if i in indices_to_correct:
            refined_translation.append(corrected_sentences[corrected_index])
            corrected_index += 1
        else:
            refined_translation.append(translation_from_Azure[i])

    return refined_translation

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import openai
import json

pair = {}
for zh, en in zip(source_sentences, reference_translations):
    pair[zh] = en

def translate_text(pair):
    openai.api_key = 'your-api-key'
    translations = []

    for zh, en in pair.items():
        messages = [
            {"role": "system", "content": "You are a Chinese to English translation corrector. You need to modify the incorrect English translations below and correct it by given Chinese sentences, please remember not to use English abbreviations and not add extra blank lines. Fix weird punctuation. And the result should be English sentences only"},
            {"role": "user", "content": json.dumps({"zh": zh, "en": en})}
        ]

        response = openai.ChatCompletion.create(
            model='gpt-4',
            messages=messages,
            max_tokens=100,
            temperature=0.7,
            timeout=30
        )

        choices = response['choices']
        if len(choices) > 0:
            model_response = choices[0]['message']['content']
            translations.append(model_response)

    return translations