【🔰Huggingface Transformers入門⑤】トークナイザーとモデルによるタスク紹介

from google.colab import drive 
drive.mount('/content/drive')
!mkdir -p '/content/drive/My Drive/huggingface_transformers_demo/'
%cd '/content/drive/My Drive/huggingface_transformers_demo/'

!git clone https://github.com/huggingface/transformers
%cd transformers
!pip install transformers

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")

sequence_0 = "The company HuggingFace is based in New York City"
# HuggingFace 社はニューヨーク市に拠点を置いています。

sequence_1 = "Apples are especially bad for your health"
# りんごは特に健康に悪い

sequence_2 = "HuggingFace's headquarters are situated in Manhattan"
# ハギングフェイスの本社はマンハッタンにあります

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")

classes = ["not paraphrase", "is paraphrase"]
# 言い換えでない、言い換えである

paraphrase = tokenizer(sequence_0, sequence_2, return_tensors="pt")
not_paraphrase = tokenizer(sequence_0, sequence_1, return_tensors="pt")

paraphrase_classification_logits = model(**paraphrase).logits
not_paraphrase_classification_logits = model(**not_paraphrase).logits

paraphrase_results = torch.softmax(paraphrase_classification_logits, dim=1).tolist()[0]
not_paraphrase_results = torch.softmax(not_paraphrase_classification_logits, dim=1).tolist()[0]

for i in range(len(classes)):
    print(f"{classes[i]}: {int(round(paraphrase_results[i] * 100))}%")

not paraphrase: 10%
is paraphrase: 90%

for i in range(len(classes)):
    print(f"{classes[i]}: {int(round(not_paraphrase_results[i] * 100))}%")

not paraphrase: 94%
is paraphrase: 6%

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

# トークナイザーとモデルをインスタンス化
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")

# 分類のクラス名を指定
classes = ["not paraphrase", "is paraphrase"]

# 比較する文章（入力）を定義
sequence_0 = "The company HuggingFace is based in New York City"
sequence_1 = "Apples are especially bad for your health"
sequence_2 = "HuggingFace's headquarters are situated in Manhattan"

# トークナイザーを使用して、2 つの文からシーケンスを作成します。
paraphrase = tokenizer(sequence_0, sequence_2, return_tensors="pt")
not_paraphrase = tokenizer(sequence_0, sequence_1, return_tensors="pt")

# シーケンスをモデルに渡す
paraphrase_classification_logits = model(**paraphrase).logits
not_paraphrase_classification_logits = model(**not_paraphrase).logits

# クラス全体の確率を取得
paraphrase_results = torch.softmax(paraphrase_classification_logits, dim=1).tolist()[0]
not_paraphrase_results = torch.softmax(not_paraphrase_classification_logits, dim=1).tolist()[0]

# sequence_0とsequence_2の比較結果を出力
print('--sequence_0,sequence_2--')
for i in range(len(classes)):
    print(f"{classes[i]}: {int(round(paraphrase_results[i] * 100))}%")

# sequence_0とsequence_1の比較結果を出力
print('--sequence_0,sequence_1--')
for i in range(len(classes)):
    print(f"{classes[i]}: {int(round(not_paraphrase_results[i] * 100))}%")

--sequence_0,sequence_2--
not paraphrase: 10%
is paraphrase: 90%
--sequence_0,sequence_1--
not paraphrase: 94%
is paraphrase: 6%

from transformers import AutoTokenizer, AutoModelForQuestionAnswering
import torch

# トークナイザーとモデルをインスタンス化
tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")

# 元の文章（入力）を定義します。
text = r"""
🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet…) for Natural Language Understanding (NLU) and Natural
Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between
TensorFlow 2.0 and PyTorch.
"""
# 質問を定義します。
questions = [
    "How many pretrained models are available in 🤗 Transformers?",
    "What does 🤗 Transformers provide?",
    "🤗 Transformers provides interoperability between which frameworks?",
]

for question in questions:
    # 質問を反復処理し、テキストと現在の質問からシーケンスを構築
    inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="pt")
    input_ids = inputs["input_ids"].tolist()[0]
    # このシーケンスをモデルに渡して、開始位置と終了位置の両方について、シーケンストークン全体 (質問とテキスト) のスコアの範囲を出力
    outputs = model(**inputs)
    answer_start_scores = outputs.start_logits
    answer_end_scores = outputs.end_logits

    # スコアのargmaxで最も可能性の高い解答の先頭を取得する
    answer_start = torch.argmax(answer_start_scores)

    # スコアのargmaxで最も可能性の高い解答の末尾を取得する
    answer_end = torch.argmax(answer_end_scores) + 1
    
    # top_kメソッドを使用して上位 5 つのトークンを取得
    answer = tokenizer.convert_tokens_to_string(
        tokenizer.convert_ids_to_tokens(input_ids[answer_start:answer_end])
    )

    print(f"Question: {question}")
    print(f"Answer: {answer}")

Question: How many pretrained models are available in 🤗 Transformers?
Answer: over 32 +
Question: What does 🤗 Transformers provide?
Answer: general - purpose architectures
Question: 🤗 Transformers provides interoperability between which frameworks?
Answer: tensorflow 2. 0 and pytorch

from transformers import AutoTokenizer, AutoModelForQuestionAnswering
import torch

# トークナイザーとモデルをインスタンス化
tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")

# 元の文章（入力）を定義します。
text = r"""
🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet…) for Natural Language Understanding (NLU) and Natural
Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between
TensorFlow 2.0 and PyTorch.
"""
# 質問を定義します。
questions = [
    "How many pretrained models are available in 🤗 Transformers?",
    "What does 🤗 Transformers provide?",
    "🤗 Transformers provides interoperability between which frameworks?",
]

for question in questions:
    # 質問を反復処理し、テキストと現在の質問からシーケンスを構築
    inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="pt")
    input_ids = inputs["input_ids"].tolist()[0]
    # このシーケンスをモデルに渡して、開始位置と終了位置の両方について、シーケンストークン全体 (質問とテキスト) のスコアの範囲を出力
    outputs = model(**inputs)
    answer_start_scores = outputs.start_logits
    answer_end_scores = outputs.end_logits

    # スコアのargmaxで最も可能性の高い解答の先頭を取得する
    answer_start = torch.argmax(answer_start_scores)

    # スコアのargmaxで最も可能性の高い解答の末尾を取得する
    answer_end = torch.argmax(answer_end_scores) + 1
    
    # top_kメソッドを使用して上位 5 つのトークンを取得
    answer = tokenizer.convert_tokens_to_string(
        tokenizer.convert_ids_to_tokens(input_ids[answer_start:answer_end])
    )

    print(f"Question: {question}")
    print(f"Answer: {answer}")

Question: How many pretrained models are available in 🤗 Transformers?
Answer: over 32 +
Question: What does 🤗 Transformers provide?
Answer: general - purpose architectures
Question: 🤗 Transformers provides interoperability between which frameworks?
Answer: tensorflow 2. 0 and pytorch

from transformers import AutoModelForMaskedLM, AutoTokenizer
import torch

# チェックポイント名からトークナイザーとモデルをインスタンス化します。
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
model = AutoModelForMaskedLM.from_pretrained("distilbert-base-cased")

# マスクされた単語の代わりに「tokenizer.mask_token」を置き、マスクされたトークンを持つシーケンスを定義します。
sequence = (
    "Distilled models are smaller than the models they mimic. Using them instead of the large "
    f"versions would help {tokenizer.mask_token} our carbon footprint."
)

# そのシーケンスをIDのリストにエンコードし、そのリストにおけるマスクされたトークンの位置を見つけます。
inputs = tokenizer(sequence, return_tensors="pt")
mask_token_index = torch.where(inputs["input_ids"] == tokenizer.mask_token_id)[1]

# マスクトークンのインデックスで予測値を取得します。
token_logits = model(**inputs).logits
mask_token_logits = token_logits[0, mask_token_index, :]

# topkメソッドを用いて上位5つのトークンを取得する。
top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()

# マスクトークンをトークンで置き換え、結果を表示する。
for token in top_5_tokens:
    print(sequence.replace(tokenizer.mask_token, tokenizer.decode([token])))

Distilled models are smaller than the models they mimic. Using them instead of the large versions would help reduce our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help increase our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help decrease our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help offset our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help improve our carbon footprint.

from transformers import AutoModelForMaskedLM, AutoTokenizer
import torch

# チェックポイント名からトークナイザーとモデルをインスタンス化。
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
model = AutoModelForMaskedLM.from_pretrained("distilbert-base-cased")

# マスクされた単語の代わりに「tokenizer.mask_token」を置き、マスクされたトークンを持つシーケンスを定義。
sequence = (
    "Distilled models are smaller than the models they mimic. Using them instead of the large "
    f"versions would help {tokenizer.mask_token} our carbon footprint."
)

# そのシーケンスをIDのリストにエンコードし、そのリストにおけるマスクされたトークンの位置を見つける。
inputs = tokenizer(sequence, return_tensors="pt")
mask_token_index = torch.where(inputs["input_ids"] == tokenizer.mask_token_id)[1]

# マスクトークンのインデックスで予測値を取得。
token_logits = model(**inputs).logits
mask_token_logits = token_logits[0, mask_token_index, :]

# topkメソッドを用いて上位5つのトークンを取得。
top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()

# マスクトークンをトークンで置き換え、結果を表示。
for token in top_5_tokens:
    print(sequence.replace(tokenizer.mask_token, tokenizer.decode([token])))

Distilled models are smaller than the models they mimic. Using them instead of the large versions would help reduce our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help increase our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help decrease our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help offset our carbon footprint.
Distilled models are smaller than the models they mimic. Using them instead of the large versions would help improve our carbon footprint.

from transformers import AutoModelForCausalLM, AutoTokenizer, top_k_top_p_filtering
import torch
from torch import nn

# チェックポイント名からトークナイザーとモデルをインスタンス化します。
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")

# シーケンスを定義します。
sequence = f"Hugging Face is based in DUMBO, New York City, and"

# そのシーケンスをIDのリストにエンコードします。
inputs = tokenizer(sequence, return_tensors="pt")
input_ids = inputs["input_ids"]

# ロジットを取得します。
next_token_logits = model(**inputs).logits[:, -1, :]

# ftop_k_top_p_filtering()によるフィルター
filtered_next_token_logits = top_k_top_p_filtering(next_token_logits, top_k=50, top_p=1.0)

# サンプルを出力
probs = nn.functional.softmax(filtered_next_token_logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)

generated = torch.cat([input_ids, next_token], dim=-1)

resulting_string = tokenizer.decode(generated.tolist()[0])
print(resulting_string)

Hugging Face is based in DUMBO, New York City, and is

from transformers import AutoTokenizer, AutoModelForCausalLM

# チェックポイント名からトークナイザーとモデルをインスタンス化。
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")

# エンコード
prompt = "Today I believe we can finally"
input_ids = tokenizer(prompt, return_tensors="pt").input_ids

# 最大 30 個のトークンを生成
outputs = model.generate(input_ids, do_sample=False, max_length=30)
tokenizer.batch_decode(outputs, skip_special_tokens=True)

[Today I believe we can finally get to the point where we can make a difference in the lives of the people of the United States of America.]
# 私たちはついにアメリカ合衆国の人々の生活に変化をもたらすことができると信じています。

from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

# チェックポイント名からトークナイザーとモデルをインスタンス化。
model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
tokenizer = AutoTokenizer.from_pretrained("t5-base")

# 入力する記事を定義。
ARTICLE = """ New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York.
A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband.
Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other.
In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage.
Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the
2010 marriage license application, according to court documents.
Prosecutors said the marriages were part of an immigration scam.
On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further.
After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective
Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002.
All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say.
Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages.
Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted.
The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s
Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali.
Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force.
If convicted, Barrientos faces up to four years in prison.  Her next court appearance is scheduled for May 18.
"""

# T5ではmax_lengthが512なので、記事を512トークンに切り詰める。
inputs = tokenizer("summarize: " + ARTICLE, return_tensors="pt", max_length=512, truncation=True)

# 結果を出力
outputs = model.generate(
    inputs["input_ids"], max_length=150, min_length=40, length_penalty=2.0, num_beams=4, early_stopping=True
)

print(tokenizer.decode(outputs[0], skip_special_tokens=True))

prosecutors say the marriages were part of an immigration scam. if convicted, barrientos faces two criminal counts of "offering a false instrument for filing in the first degree" she has been married 10 times, nine of them between 1999 and 2002.

from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

# チェックポイント名からトークナイザーとモデルをインスタンス化
model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
tokenizer = AutoTokenizer.from_pretrained("t5-base")

# T5特有のプレフィックス「translate English to German:」を追加
inputs = tokenizer(
    "translate English to German: Hugging Face is a technology company based in New York and Paris",
    return_tensors="pt",
)

# PreTrainedModel.generate()メソッドを使用して翻訳を実行
outputs = model.generate(inputs["input_ids"], max_length=40, num_beams=4, early_stopping=True)

print(tokenizer.decode(outputs[0], skip_special_tokens=True))

Hugging Face ist ein Technologieunternehmen mit Sitz in New York und Paris.