【🔰Huggingface Transformers入門⑧】文章分類モデルを作成する（3）　〜モデルの評価と推論〜

from google.colab import drive 
drive.mount('/content/drive')
!mkdir -p '/content/drive/My Drive/huggingface_transformers_demo/'
%cd '/content/drive/My Drive/huggingface_transformers_demo/'

!pip install transformers fugashi ipadic sentencepiece datasets

import torch
import pandas as pd
import numpy as np
from transformers import AutoModelForSequenceClassification, AutoTokenizer

# トークナイザの取得
tokenizer = AutoTokenizer.from_pretrained('./text/news_model')

# モデルの取得
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = (AutoModelForSequenceClassification
    .from_pretrained('./text/news_model')
    .to(device))

test_df = pd.read_csv("./text/news_test.csv",encoding='UTF-8')
test_df[:10]

test_df = pd.read_csv("./text/news_test.csv",encoding='UTF-8')
test_df[:10]

sample_text = test_df['sentence'][1]
sample_text

# Interop Tokyo 2012：デジタルサイネージジャパンにてセラクがAndroidを搭載した鏡付き洗面台「スマート洗面台」を出展【レポート】

inputs = tokenizer(sample_text, return_tensors="pt")
labels =['dokujo', 'it', 'sports-','kaden','homme','movie','peachy','smax','topic-news']
model.eval()

with torch.no_grad():
    outputs = model(inputs["input_ids"].to(device), inputs["attention_mask"].to(device))
    prediction = torch.nn.functional.softmax(outputs.logits,dim=1)

#ラベル番号を表示
print(int(torch.argmax(prediction)))

#ラベル名を表示
print(labels[int(torch.argmax(prediction))])

#ラベル番号
7

#ラベル名
smax

print(prediction.cpu().detach().numpy())

#[[0.00333388 0.01006817 0.00436237 0.06067429 0.00629614 0.00435802 0.00476074 0.9016149  0.00453156]]

print(np.max(prediction.cpu().detach().numpy()))

# 0.9016149

df = pd.DataFrame(columns=['label','pred', 'sentence'])
for sentence in test_df['sentence']:
  inputs = tokenizer(sentence, return_tensors="pt")
  labels =['dokujo', 'it', 'sports-','kaden','homme','movie','peachy','smax','topic-news']
  model.eval()

  with torch.no_grad():
      outputs = model(inputs["input_ids"].to(device), inputs["attention_mask"].to(device))
      prediction = torch.nn.functional.softmax(outputs.logits,dim=1)
      df = df.append({'label':labels[int(torch.argmax(prediction))] ,'pred':np.max(prediction.cpu().detach().numpy()),'sentence':sentence}, ignore_index=True)

df.to_csv("./text/news_test_result.csv",encoding='UTF-8')

y_true = []
y_pred = []

for item in zip(test_df['label'], test_df['sentence']):
  inputs = tokenizer(item[1], return_tensors="pt")
  labels =['dokujo', 'it', 'sports-','kaden','homme','movie','peachy','smax','topic-news']
  model.eval()

  with torch.no_grad():
      outputs = model(inputs["input_ids"].to(device), inputs["attention_mask"].to(device))
      prediction = torch.nn.functional.softmax(outputs.logits,dim=1)
      y_true.append(item[0])
      y_pred.append(int(torch.argmax(prediction)))

labels =['dokujo', 'it', 'sports-','kaden','homme','movie','peachy','smax','topic-news']

def plot_confusion_matrix(y_preds, y_true, labels):
    cm = confusion_matrix(y_true, y_pred, normalize="true")
    fig, ax = plt.subplots(figsize=(6, 6))
    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)
    disp.plot(cmap="Blues", values_format=".2f", ax=ax, colorbar=False)
    plt.title("Normalized confusion matrix")
    plt.show()

plot_confusion_matrix(y_pred, y_true,labels)

from sklearn.metrics import classification_report
labels =['dokujo', 'it', 'sports-','kaden','homme','movie','peachy','smax','topic-news']
report = classification_report(y_true, y_pred,target_names=labels,output_dict=True)

report_df = pd.DataFrame(report).T
report_df