> ls
face_image.py
data

> ls data
CACD2000
celebrity2000_meta.mat

import h5py #HDF5を扱うライブラリ
from PIL import Image #画像の表示/保存/書き込みなどを扱うライブラリ
import os
import numpy as np
import scipy.io

#画像データの保存先
image_dir = 'data/CACD2000'

# .matファイルの読み込み（古い形式の場合）
## 辞書型として読み込まれる
file = scipy.io.loadmat('data/celebrity2000_meta.mat')

# 辞書のKeyを表示する
print('keys:',file.keys())
# >>> dict_keys(['__header__', '__version__', '__globals__', 'celebrityData', 'celebrityImageData'])

#celebrityImageDataの確認
print(file['celebrityImageData'])

"""
[[(array([[53],
         [53],
         [53],
         ...,
         [23],
         [23],
         [23]], dtype=uint8), array([[   1],
         [   1],
         [   1],
         ...,
         [2000],
         [2000],
         [2000]], dtype=uint16), array([[2004],
         [2004],
         [2004],
         ...,
         [2013],
         [2013],
         [2013]], dtype=uint16), array([], shape=(0, 0), dtype=uint8), array([[ 1],
         [ 1],
         [ 1],
         ...,
         [50],
         [50],
         [50]], dtype=uint8), array([[1],
         [1],
         [1],
         ...,
         [0],
         [0],
         [0]], dtype=uint8), array([[1951],
         [1951],
         [1951],
         ...,
         [1990],
         [1990],
         [1990]], dtype=uint16), array([[array(['53_Robin_Williams_0001.jpg'], dtype='<U26')],
         [array(['53_Robin_Williams_0002.jpg'], dtype='<U26')],
         [array(['53_Robin_Williams_0003.jpg'], dtype='<U26')],
         ...,
         [array(['23_Katie_Findlay_0011.jpg'], dtype='<U25')],
         [array(['23_Katie_Findlay_0012.jpg'], dtype='<U25')],
         [array(['23_Katie_Findlay_0013.jpg'], dtype='<U25')]], dtype=object))                ]]
"""
# 7個目に画像のファイル名が入っているので
# celebrityImageDataから画像ファイル名を抽出
image_data = file['celebrityImageData']
jpg_files = [str(image_name[0][0]) for image_name in image_data[0][0][7]]

# 抽出された.jpgファイル名のリストを上から10個表示
print(jpg_files[:10])

# 画像データの取得と表示
## 名前データを利用して画像をいくつか開いてみます.
for n in jpg_files[:10]:
    img_path = os.path.join(image_dir, n)  # パスを結合し,ファイル名を取得

    if os.path.exists(img_path):
        img = Image.open(img_path)  # 画像ファイルを開く
        img.show()  # 画像を表示
    else:
        print(f"Image file not found: {img_path}")
    #>>> 画像が表示されます

import os
import shutil
import scipy.io
from collections import defaultdict
import random

# 画像ディレクトリの設定
image_dir = 'data/CACD2000'
output_dir = 'data/sorted_images'

# .matファイルの読み込み
file = scipy.io.loadmat('data/celebrity2000_meta.mat')

# celebrityImageDataから年齢と画像ファイル名を抽出
image_data = file['celebrityImageData']
# 年齢情報
ages = image_data[0][0][0].flatten()
# 画像ファイル名
jpg_files = [str(image_name[0][0]) for image_name in image_data[0][0][7]]

# 年齢と画像ファイルをペアにする
age_image_pairs = list(zip(ages, jpg_files))

# 年代ごとの画像カウント
age_group_counts = defaultdict(int)

# 年齢別に画像をシャッフル
random.shuffle(age_image_pairs)

# 年齢別のフォルダに画像をコピー（各年代最大200枚）
for age, jpg_file in age_image_pairs:
    age_group = (age // 10) * 10
    if age_group > 100:
        age_group = 100  # 100代以上は100代フォルダに保存

    # 各年代ごとに200枚までコピー
    if age_group_counts[age_group] < 200:
        folder_path = os.path.join(output_dir, f'{age_group}s')
        os.makedirs(folder_path, exist_ok=True)

        src_path = os.path.join(image_dir, jpg_file)
        dst_path = os.path.join(folder_path, jpg_file)

        shutil.copy(src_path, dst_path)
        age_group_counts[age_group] += 1

> ls data/sorted_images
10s 20s 30s 40s 50s 60s
> ls data/sorted_images/10s |head -20
19_Alison_Pill_0001.jpg
19_Alison_Pill_0002.jpg
19_Alison_Pill_0003.jpg
19_Alison_Pill_0005.jpg
19_Alison_Pill_0006.jpg
19_Alison_Pill_0007.jpg
19_Alison_Pill_0009.jpg
19_Alison_Pill_0011.jpg
19_Amanda_Seyfried_0001.jpg
19_Amanda_Seyfried_0002.jpg
19_Amanda_Seyfried_0004.jpg
19_Amanda_Seyfried_0005.jpg
19_Amanda_Seyfried_0007.jpg
19_Amanda_Seyfried_0008.jpg
19_Amanda_Seyfried_0010.jpg
19_Amanda_Seyfried_0011.jpg
19_Amanda_Seyfried_0013.jpg
19_Amanda_Seyfried_0014.jpg
19_Anna_Kendrick_0002.jpg
19_Anna_Kendrick_0008.jpg

import os
import shutil
from sklearn.model_selection import train_test_split

data_dir = 'data/sorted_images'
output_dir = 'data/sorted_images_split'

# 画像ファイルのパスを収集し,年齢別に分類
age_groups = ['10s', '20s', '30s', '40s', '50s', '60s']
for age_group in age_groups:
    images = os.listdir(os.path.join(data_dir, age_group))
    train_images, val_images = train_test_split(images
                                               ,test_size=0.2 #2割をテスト用データにする
                                               , random_state=42)

    train_dir = os.path.join(output_dir, 'train', age_group)
    val_dir = os.path.join(output_dir, 'val', age_group)
    os.makedirs(train_dir, exist_ok=True)
    os.makedirs(val_dir, exist_ok=True)

    for image in train_images:
        shutil.copy(os.path.join(data_dir, age_group, image), os.path.join(train_dir, image))
    for image in val_images:
        shutil.copy(os.path.join(data_dir, age_group, image), os.path.join(val_dir, image))

data/sorted_images_split
├── train
│      ├── 10s
│      ├── 20s
│      ├── 30s
│      ├── 40s
│      ├── 50s
│      └── 60s
└── val
        ├── 10s
        ├── 20s
        ├── 30s
        ├── 40s
        ├── 50s
        └── 60s

❯ tree -d data/result
data/result
├── feature_maps
│   ├── 10s
│   │   ├── features_-1
│   │   ├── features_0
│   │   ├── features_5
│   │   └── last_conv
│   ├── 20s
│   │   ├── features_-1
│   │   ├── features_0
│   │   ├── features_5
│   │   └── last_conv
│   ├── 30s
│   │   ├── features_-1
│   │   ├── features_0
│   │   ├── features_5
│   │   └── last_conv
│   ├── 40s
│   │   ├── features_-1
│   │   ├── features_0
│   │   ├── features_5
│   │   └── last_conv
│   ├── 50s
│   │   ├── features_-1
│   │   ├── features_0
│   │   ├── features_5
│   │   └── last_conv
│   └── 60s
│       ├── features_-1
│       ├── features_0
│       ├── features_5
│       └── last_conv

#------------------------------------------------------------------
#特徴マップの描画
#------------------------------------------------------------------
# 中間層のフックを設定する層リストの設定例
target_layers_indices = [0, 5, -1]  # お好みで変更可能

# フックで取り出した特徴マップを保持する辞書
intermediate_feature_maps = {}

def get_intermediate_hook(name):
    def hook_fn(m, input, output):
        # output: [B, C, H, W]
        intermediate_feature_maps[name] = output.detach().cpu()
    return hook_fn

# 対象層にフックを登録
hooks = []
for idx in target_layers_indices:
    layer_name = f"features_{idx}"
    h = model.features[idx].register_forward_hook(get_intermediate_hook(layer_name))
    hooks.append(h)

# --- 最後の畳み込み層のフック ---
feature_maps_last_conv = []
def get_feature_map_hook_conv(self, input, output):
    feature_maps_last_conv.append(output.detach().cpu())

hook_conv = model.features[-1].register_forward_hook(get_feature_map_hook_conv)


# 抽出したいクラス (0:10s,1:20s,...,5:60s)
target_classes = [0,1,2,3,4,5]
# 各クラスから9枚ずつ取得するためのカウンタ
images_per_class = 9
extracted_counts = {cls: 0 for cls in target_classes}

# 保存ディレクトリの用意
base_dir = 'data/result/feature_maps'
if not os.path.exists(base_dir):
    os.makedirs(base_dir)

# クラスごとのフォルダを作成 (10s～60s)
for cls in target_classes:
    class_label_str = f"{(cls+1)*10}s"
    class_dir = os.path.join(base_dir, class_label_str)
    if not os.path.exists(class_dir):
        os.makedirs(class_dir)
    # 中間層用のサブフォルダを用意
    for idx in target_layers_indices:
        layer_name = f"features_{idx}"
        layer_dir = os.path.join(class_dir, layer_name)
        if not os.path.exists(layer_dir):
            os.makedirs(layer_dir)

    # 最終Conv層用のフォルダ
    last_conv_dir = os.path.join(class_dir, 'last_conv')
    if not os.path.exists(last_conv_dir):
        os.makedirs(last_conv_dir)

model.eval()
val_iter = iter(dataloaders['val'])
with torch.no_grad():
    # 全クラスが指定枚数分揃うまで繰り返す
    while not all(count == images_per_class for count in extracted_counts.values()):
        try:
            inputs, labels = next(val_iter)
        except StopIteration:
            # データがなくなったら終了
            break

        inputs = inputs.to(device)
        # フック用リストをクリア
        feature_maps_last_conv.clear()
        intermediate_feature_maps.clear()

        # 順伝播：フックで特徴マップ取得
        outputs = model(inputs)

        fm_batch = feature_maps_last_conv[0]   # [B, C, H, W] 最終Conv層出力

        for i in range(inputs.size(0)):
            cls = labels[i].item()
            if cls in target_classes and extracted_counts[cls] < images_per_class:

                class_label_str = f"{(cls+1)*10}s"
                class_dir = os.path.join(base_dir, class_label_str)

                # 中間層特徴マップ可視化
                for idx in target_layers_indices:
                    layer_name = f"features_{idx}"
                    fm_intermediate = intermediate_feature_maps[layer_name][i] # shape: [C, H, W]
                    num_channels_to_plot = min(8, fm_intermediate.shape[0])

                    fig, axes = plt.subplots(2, 4, figsize=(12, 6))
                    axes = axes.flatten()
                    for ch in range(num_channels_to_plot):
                        ax = axes[ch]
                        ax.imshow(fm_intermediate[ch].numpy(), cmap='viridis')
                        ax.axis('off')
                    plt.suptitle(f'Intermediate Layer {layer_name} Feature Maps: Class {(cls+1)*10}s (Image {extracted_counts[cls]+1})')

                    layer_dir = os.path.join(class_dir, layer_name)
                    out_path = os.path.join(layer_dir, f'feature_map_{extracted_counts[cls]+1}.png')
                    plt.savefig(out_path)
                    plt.close()

                # 最終Conv層特徴マップ可視化
                fm = fm_batch[i]  # shape: [C, H, W]
                num_channels_to_plot = min(8, fm.shape[0])
                fig, axes = plt.subplots(2, 4, figsize=(12, 6))
                axes = axes.flatten()
                for ch in range(num_channels_to_plot):
                    ax = axes[ch]
                    ax.imshow(fm[ch].numpy(), cmap='viridis')
                    ax.axis('off')
                plt.suptitle(f'Last Conv Feature Maps: Class {(cls+1)*10}s (Image {extracted_counts[cls]+1})')
                last_conv_dir = os.path.join(class_dir, 'last_conv')
                out_path = os.path.join(last_conv_dir, f'feature_map_{extracted_counts[cls]+1}.png')
                plt.savefig(out_path)
                plt.close()

                extracted_counts[cls] += 1

                if all(count == images_per_class for count in extracted_counts.values()):
                    break

# フックを削除
hook_conv.remove()
for h in hooks:
    h.remove()
print('特徴マップ描画完了')

#------------------------------------------------------------------
# Grad-CAM, Guided Backprop, Guided Grad-CAM の実行 (10s, 60sに限定)
#------------------------------------------------------------------
import cv2

# 対象クラスIDとクラス名
target_cam_classes = {0:'10s', 5:'60s'}

# valデータからサンプル画像を取得
base_val_dir = os.path.join(data_dir, 'val')
sample_images = {}
for cls_id, cls_name in target_cam_classes.items():
    cls_dir = os.path.join(base_val_dir, cls_name)
    img_name = os.listdir(cls_dir)[0]
    img_path = os.path.join(cls_dir, img_name)
    sample_images[cls_id] = img_path

# Grad-CAM用フック設定
target_layer = model.features[-1]
features = None
gradients = None

def forward_hook(module, input, output):
    nonlocal features
    features = output

def backward_hook(module, grad_input, grad_output):
    nonlocal gradients
    gradients = grad_output[0]

forward_h = target_layer.register_forward_hook(forward_hook)
backward_h = target_layer.register_full_backward_hook(backward_hook)

# Guided Backprop用：ConvNeXtはGELUを使用, そのためGELUに対するGuided Backpropを実装
# 出力が正の部分のみ勾配を通す
gelu_outputs = {}  # moduleをキーにしてforward出力を保存

def gelu_forward_hook(module, input, output):
    gelu_outputs[module] = output

def gelu_backward_hook(module, grad_input, grad_output):
    # grad_input: tuple of gradients wrt input of gelu
    # grad_output: tuple of gradients wrt output of gelu
    # guided backprop: 出力が正の位置のみ勾配を通す
    out = gelu_outputs[module]
    # outと同じ形状で, out>0のとこだけ1, それ以外0
    positive_mask = (out > 0).float()
    # grad_output[0]に対して, positive_maskをかけて負の領域をカット
    guided_grad = grad_output[0] * positive_mask
    return (guided_grad,)

# GELU層にフックを登録
guided_hooks = []
for m in model.modules():
    if isinstance(m, nn.GELU):
        fh = m.register_forward_hook(gelu_forward_hook)
        bh = m.register_backward_hook(gelu_backward_hook)
        guided_hooks.append(fh)
        guided_hooks.append(bh)

def preprocess_image(img_path):
    img = Image.open(img_path).convert('RGB')
    img = img.resize((224, 224))
    img_tensor = data_transforms['val'](img).unsqueeze(0).to(device)
    return img, img_tensor

def generate_gradcam():
    pooled_gradients = torch.mean(gradients, dim=[0,2,3])
    cam = torch.zeros(features.shape[2:], dtype=features.dtype, device=features.device)
    for i in range(features.shape[1]):
        cam += pooled_gradients[i] * features[0,i,:,:]
    cam = cam.cpu().data.numpy()
    cam = np.maximum(cam, 0)
    if np.max(cam) != 0:
        cam = cam / np.max(cam)
    return cam

def do_guided_backprop(model, img_tensor, target_class):
    # 勾配リセット
    model.zero_grad()
    img_tensor.grad = None
    # forward
    output = model(img_tensor)
    loss = output[0, target_class]
    model.zero_grad()
    loss.backward()
    guided_grad = img_tensor.grad.data[0].cpu().numpy().transpose(1,2,0)
    guided_grad = guided_grad - guided_grad.min()
    guided_grad = guided_grad / (guided_grad.max() + 1e-8)
    return guided_grad

def apply_colormap_on_image(org_img, cam, alpha=0.5):
    H_org, W_org, _ = org_img.shape
    cam_resized = cv2.resize(cam, (W_org, H_org))

    heatmap = cv2.applyColorMap(np.uint8(255*cam_resized), cv2.COLORMAP_JET)
    heatmap = np.float32(heatmap)/255.0
    org_img = np.float32(org_img)/255.0

    cam_img = heatmap * alpha + org_img
    cam_img = cam_img / np.max(cam_img)
    return np.uint8(255*cam_img)

def guided_gradcam(guided_grad, cam):
    H, W, _ = guided_grad.shape
    cam_resized = cv2.resize(cam, (W, H))
    guided_gradcam = guided_grad * cam_resized[..., np.newaxis]
    guided_gradcam = guided_gradcam - guided_gradcam.min()
    guided_gradcam = guided_gradcam / (guided_gradcam.max()+1e-8)
    return guided_gradcam

gradcam_dir = 'data/result/gradcam_guided'
if not os.path.exists(gradcam_dir):
    os.makedirs(gradcam_dir)

model.eval()
from PIL import Image
for cls_id, img_path in sample_images.items():
    # original画像読み込み
    org_img, img_tensor = preprocess_image(img_path)
    org_img_np = np.array(org_img)  # RGB, 224x224
    img_tensor.requires_grad = True

    # Grad-CAM
    model.zero_grad()
    output = model(img_tensor)
    target_score = output[0, cls_id]
    target_score.backward()
    cam = generate_gradcam()

    # Grad-CAM画像
    org_img_cv = org_img_np[:,:,::-1].copy()
    gradcam_img = apply_colormap_on_image(org_img_cv, cam)
    gradcam_img_rgb = gradcam_img[:,:,::-1]

    # Guided Backprop
    gb = do_guided_backprop(model, img_tensor, cls_id) # 0-1 float
    g_gradcam = guided_gradcam(gb, cam) # 0-1 float

    original_uint8 = org_img_np
    gradcam_uint8 = gradcam_img_rgb
    gb_uint8 = (gb*255).astype(np.uint8)
    g_gradcam_uint8 = (g_gradcam*255).astype(np.uint8)

    combined = np.hstack([original_uint8, gradcam_uint8, gb_uint8, g_gradcam_uint8])
    plt.imsave(os.path.join(gradcam_dir, f'class_{cls_id}_combined.png'), combined)

forward_h.remove()
backward_h.remove()
for h in guided_hooks:
    h.remove()

print("Grad-CAM, Guided Backprop, Guided Grad-CAM 完了")

import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import random

# 次元削減用
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA

# CNN用
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms, models
from torch.utils.data import DataLoader
from torchvision.models import ConvNeXt_Tiny_Weights


#学習した特徴量を抽出する
def extract_features(model, dataloader, device):
    model.eval()
    features = []
    labels_list = []

    with torch.no_grad():
        for inputs, labels in dataloader:
            inputs = inputs.to(device)
            labels = labels.to(device)
            outputs = model(inputs)
            features.append(outputs.cpu())
            labels_list.append(labels.cpu())

    features = torch.cat(features, dim=0)
    labels = torch.cat(labels_list, dim=0)

    return features, labels

# t-SNEによる次元圧縮と散布図の描画
def plot_tsne(train_features, train_labels, path):
    # train_featuresとtrain_labelsをnumpyに変換
    features_np = train_features.numpy()
    labels_np = train_labels.numpy()

    # t-SNEによる次元圧縮 (2次元)
    tsne = TSNE(n_components=2, random_state=42)
    features_2d = tsne.fit_transform(features_np)

    # 散布図の描画
    plt.figure(figsize=(10, 8))

    # 各ラベルに基づいて色分けしてプロット
    num_classes = len(np.unique(labels_np))  # クラス数を取得
    for label in np.unique(labels_np):
        indices = np.where(labels_np == label)
        plt.scatter(features_2d[indices, 0]
                   ,features_2d[indices, 1]
                   ,label=f'{(label + 1) * 10}s'
                   ,alpha=0.6)

    plt.title('t-SNE of Train Features')
    plt.xlabel('t-SNE Component 1')
    plt.ylabel('t-SNE Component 2')
    plt.legend(title="Age Group")
    plt.grid(True)
    plt.savefig(path)
    plt.close()

# PCAによる次元圧縮と散布図の描画
def plot_pca(train_features, train_labels, path):
    # train_featuresとtrain_labelsをnumpyに変換
    features_np = train_features.numpy()
    labels_np = train_labels.numpy()

    # PCAによる次元圧縮 (2次元)
    pca = PCA(n_components=2)
    features_2d = pca.fit_transform(features_np)

    # 散布図の描画
    plt.figure(figsize=(10, 8))

    # 各ラベルに基づいて色分けしてプロット
    num_classes = len(np.unique(labels_np))  # クラス数を取得
    for label in np.unique(labels_np):
        indices = np.where(labels_np == label)
        plt.scatter(features_2d[indices, 0]
                   ,features_2d[indices, 1]
                   ,label=f'{(label + 1) * 10}s'
                   ,alpha=0.6)

    plt.title('PCA of Train Features')
    plt.xlabel('PCA Component 1')
    plt.ylabel('PCA Component 2')
    plt.legend(title="Age Group")
    plt.grid(True)
    plt.savefig(path)
    plt.close()

# 乱数シードを設定
def set_seed(seed):
    torch.manual_seed(seed)
    np.random.seed(seed)              # Numpy用の乱数シードを設定
    random.seed(seed)                 # Pythonの標準乱数シードを設定

    # 再現性を完全に保証するために以下も設定（ただし、若干のパフォーマンス低下の可能性あり）
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

def main():
    # シードを設定する
    #(自分の研究でやる場合は以下の行は消しても問題ない.)
    set_seed(2024)
    # データのディレクトリ設定
    data_dir = 'data/sorted_images_split'
    batch_size = 32
    num_epochs = 5
    num_classes = 6  # 10代, 20代, ..., 60代

    # デバイス設定
    #GPUが利用できる場合はGPUを使う,そうでない場合はCPUを計算に利用します.
    if torch.backends.mps.is_available():
        device = torch.device("mps") #Mac GPU
    elif torch.cuda.is_available():
        device = torch.device("cuda:0") #Win GPU
    else:
        device = torch.device("cpu") #CPU

    print(f'Using device: {device}')

    # モデル定義
    # ConvNextモデルの読み込みとカスタマイズ
    weights = ConvNeXt_Tiny_Weights.IMAGENET1K_V1  # 最新の重みを指定
    model = models.convnext_tiny(weights=weights)  # ConvNextの小さいモデルを使用
    #モデル分類層の最終層(第3層(0,1,2番目))の入力特徴量を取得
    num_ftrs = model.classifier[2].in_features
    #既に学習されたモデルではクラス数がことなるので ,入力特徴量の数(num_ftrs)はそのまま
    #出力をクラス数に変更
    model.classifier[2] = nn.Linear(num_ftrs, num_classes)
    model = model.to(device)
    # データ変換（前処理）
    data_transforms = {
        'train': transforms.Compose([
            transforms.Resize((224, 224)),     #画像のリサイズ
            transforms.RandomHorizontalFlip(), #画像をランダムに反転
            transforms.ColorJitter(brightness=0.2, contrast=0.2),  # 色調変化
            transforms.ToTensor(),             #テンソル(多次元配列)に変換
            transforms.Normalize(mean=[0.485, 0.456, 0.406]
                                ,std=[0.229, 0.224, 0.225]), #正規化(本来は値を変更する必要あり.
        ]),
        'val': transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406]
                                ,[0.229, 0.224, 0.225]),
        ]),
    }

    # データセットの読み込み
    image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                              data_transforms[x])
                      for x in ['train', 'val']}
    dataloaders = {x: DataLoader(image_datasets[x]
                                ,batch_size=batch_size
                                ,shuffle=True
                                ,num_workers=4) #使用するCore数
                   for x in ['train', 'val']}

    # 毎回学習するの大変なので,学習済みの場合は結果を読み込む
    if os.path.exists('data/result/trained_model_weights.pth'):
        print("学習済みデータのロード中")
        model.load_state_dict(torch.load('data/result/trained_model_weights.pth', map_location=device))
    else:
        print("学習開始")
        # 損失関数とオプティマイザ
        criterion = nn.CrossEntropyLoss() #クロスエントロピー損失
        optimizer = optim.Adam(model.parameters() #Adam
                              ,lr=0.0001) #Learning rate (学習率)

        #SGDを利用する場合
        #optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

        #結果の記録用
        train_losses = []
        train_accuracies = []
        val_losses = []
        val_accuracies = []
        results = []

        # 学習ループ
        for epoch in range(num_epochs):
            print(f'Epoch {epoch+1}/{num_epochs}')
            print('-' * 10)

            for phase in ['train', 'val']:
                if phase == 'train':
                    model.train()
                else:
                    model.eval()

                running_loss = 0.0
                running_corrects = 0

                for inputs, labels in dataloaders[phase]:
                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    optimizer.zero_grad()

                    with torch.set_grad_enabled(phase == 'train'):
                        outputs = model(inputs)
                        _, preds = torch.max(outputs, 1)
                        loss = criterion(outputs, labels)

                        if phase == 'train':
                            loss.backward()
                            optimizer.step()

                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data)
                    if phase == 'val':  # バリデーション時に予測と実際のラベルを保存
                        for i in range(len(labels)):
                            results.append({
                                'epoch': epoch + 1,
                                'pred': preds[i].item(),
                                'acctual': labels[i].item()
                            })

                epoch_loss = running_loss / len(image_datasets[phase])
                epoch_acc = running_corrects.float() / len(image_datasets[phase])

                print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')
                if phase == 'train':
                    train_losses.append(epoch_loss)
                    train_accuracies.append(epoch_acc.item())
                else:
                    val_losses.append(epoch_loss)
                    val_accuracies.append(epoch_acc.item())

        #------------------------------------------------------------------
        #結果の表示
        #------------------------------------------------------------------

        print('Training complete')
        print("Training Losses: ", train_losses)
        print("Training Accuracies: ", train_accuracies)
        print("Validation Losses: ", val_losses)
        print("Validation Accuracies: ", val_accuracies)

        # 結果を可視化してCSVファイルに保存
        loss_acc = pd.DataFrame({'train_losses':train_losses
                                ,'train_accuracies':train_accuracies
                                ,'val_losses':val_losses
                                ,'val_accuracies':val_accuracies})
        plt.title('Losses')
        plt.xlabel('Epoch')
        plt.ylabel('Losses')
        plt.plot(np.arange(num_epochs),loss_acc['train_losses'],c='r',label='train_losses')
        plt.plot(np.arange(num_epochs),loss_acc['val_losses'],c='b',label='val_losses')
        plt.grid(True)
        plt.legend()
        plt.savefig('data/result/convnext_loss.png')
        plt.close()

        plt.title('Accuracies')
        plt.xlabel('Epoch')
        plt.ylabel('Accuracies')
        plt.plot(np.arange(num_epochs),loss_acc['train_accuracies'],c='r',label='train_accuracies')
        plt.plot(np.arange(num_epochs),loss_acc['val_accuracies'],c='b',label='val_accuracies')
        plt.grid(True)
        plt.legend()
        plt.savefig('data/result/convnext_acc.png')
        plt.close()

        loss_acc.to_csv('data/result/convnext_loss_acc.csv'
                       ,encoding='utf_8_sig')

        results_df = pd.DataFrame(results)
        result_max_epochs = results_df[results_df['epoch'] == num_epochs]
        result_heatmap = pd.DataFrame(index=np.arange(6)
                                     ,columns=np.arange(6)
                                     ,data=0)

        #実際のラベルに対する予測された回数をカウント
        for i in result_max_epochs.index:
            p = result_max_epochs.at[i,'pred']
            a = result_max_epochs.at[i,'acctual']
            result_heatmap.at[p,a] +=1

        #列相対度数に変換
        for c in result_heatmap:
            result_heatmap[c] = result_heatmap[c] / result_heatmap[c].sum()

        #ヒートマップとして表現
        sns.heatmap(result_heatmap
                   ,annot=True)
        plt.ylabel('pred')
        plt.xlabel('acctual')
        plt.savefig('data/result/pred_acctual_heatmap.png')
        plt.close()
        results_df.to_csv('data/result/pred_acctual.csv'
                         ,encoding='utf_8_sig')

        # 特徴量を取得する
        train_features, train_labels = extract_features(model, dataloaders['train'], device)
        val_features, val_labels = extract_features(model, dataloaders['val'], device)


        #散布図の描画
        plot_tsne(train_features, train_labels,'data/result/convnext_tsne.png')
        plot_pca(train_features, train_labels,'data/result/convnext_pca.png')

        # (毎回学習するのは大変なので)特徴量を表示または保存しておく
        torch.save(model.state_dict(), 'data/result/trained_model_weights.pth')

    #------------------------------------------------------------------
    #特徴マップの描画
    #------------------------------------------------------------------
    # 中間層のフックを設定する層リストの設定例
    target_layers_indices = [0, 5, -1]  # お好みで変更可能

    # フックで取り出した特徴マップを保持する辞書
    intermediate_feature_maps = {}

    def get_intermediate_hook(name):
        def hook_fn(m, input, output):
            # output: [B, C, H, W]
            intermediate_feature_maps[name] = output.detach().cpu()
        return hook_fn

    # 対象層にフックを登録
    hooks = []
    for idx in target_layers_indices:
        layer_name = f"features_{idx}"
        h = model.features[idx].register_forward_hook(get_intermediate_hook(layer_name))
        hooks.append(h)

    # --- 最後の畳み込み層のフック ---
    feature_maps_last_conv = []
    def get_feature_map_hook_conv(self, input, output):
        feature_maps_last_conv.append(output.detach().cpu())

    hook_conv = model.features[-1].register_forward_hook(get_feature_map_hook_conv)


    # 抽出したいクラス (0:10s,1:20s,...,5:60s)
    target_classes = [0,1,2,3,4,5]
    # 各クラスから9枚ずつ取得するためのカウンタ
    images_per_class = 9
    extracted_counts = {cls: 0 for cls in target_classes}

    # 保存ディレクトリの用意
    base_dir = 'data/result/feature_maps'
    if not os.path.exists(base_dir):
        os.makedirs(base_dir)

    # クラスごとのフォルダを作成 (10s～60s)
    for cls in target_classes:
        class_label_str = f"{(cls+1)*10}s"
        class_dir = os.path.join(base_dir, class_label_str)
        if not os.path.exists(class_dir):
            os.makedirs(class_dir)
        # 中間層用のサブフォルダを用意
        for idx in target_layers_indices:
            layer_name = f"features_{idx}"
            layer_dir = os.path.join(class_dir, layer_name)
            if not os.path.exists(layer_dir):
                os.makedirs(layer_dir)

        # 最終Conv層用のフォルダ
        last_conv_dir = os.path.join(class_dir, 'last_conv')
        if not os.path.exists(last_conv_dir):
            os.makedirs(last_conv_dir)

    model.eval()
    val_iter = iter(dataloaders['val'])
    with torch.no_grad():
        # 全クラスが指定枚数分揃うまで繰り返す
        while not all(count == images_per_class for count in extracted_counts.values()):
            try:
                inputs, labels = next(val_iter)
            except StopIteration:
                # データがなくなったら終了
                break

            inputs = inputs.to(device)
            # フック用リストをクリア
            feature_maps_last_conv.clear()
            intermediate_feature_maps.clear()

            # 順伝播：フックで特徴マップ取得
            outputs = model(inputs)

            fm_batch = feature_maps_last_conv[0]   # [B, C, H, W] 最終Conv層出力

            for i in range(inputs.size(0)):
                cls = labels[i].item()
                if cls in target_classes and extracted_counts[cls] < images_per_class:

                    class_label_str = f"{(cls+1)*10}s"
                    class_dir = os.path.join(base_dir, class_label_str)

                    # 中間層特徴マップ可視化
                    for idx in target_layers_indices:
                        layer_name = f"features_{idx}"
                        fm_intermediate = intermediate_feature_maps[layer_name][i] # shape: [C, H, W]
                        num_channels_to_plot = min(8, fm_intermediate.shape[0])

                        fig, axes = plt.subplots(2, 4, figsize=(12, 6))
                        axes = axes.flatten()
                        for ch in range(num_channels_to_plot):
                            ax = axes[ch]
                            ax.imshow(fm_intermediate[ch].numpy(), cmap='viridis')
                            ax.axis('off')
                        plt.suptitle(f'Intermediate Layer {layer_name} Feature Maps: Class {(cls+1)*10}s (Image {extracted_counts[cls]+1})')

                        layer_dir = os.path.join(class_dir, layer_name)
                        out_path = os.path.join(layer_dir, f'feature_map_{extracted_counts[cls]+1}.png')
                        plt.savefig(out_path)
                        plt.close()

                    # 最終Conv層特徴マップ可視化
                    fm = fm_batch[i]  # shape: [C, H, W]
                    num_channels_to_plot = min(8, fm.shape[0])
                    fig, axes = plt.subplots(2, 4, figsize=(12, 6))
                    axes = axes.flatten()
                    for ch in range(num_channels_to_plot):
                        ax = axes[ch]
                        ax.imshow(fm[ch].numpy(), cmap='viridis')
                        ax.axis('off')
                    plt.suptitle(f'Last Conv Feature Maps: Class {(cls+1)*10}s (Image {extracted_counts[cls]+1})')
                    last_conv_dir = os.path.join(class_dir, 'last_conv')
                    out_path = os.path.join(last_conv_dir, f'feature_map_{extracted_counts[cls]+1}.png')
                    plt.savefig(out_path)
                    plt.close()

                    extracted_counts[cls] += 1

                    if all(count == images_per_class for count in extracted_counts.values()):
                        break

    # フックを削除
    hook_conv.remove()
    for h in hooks:
        h.remove()
    print('特徴マップ描画完了')

    #------------------------------------------------------------------
    # Grad-CAM, Guided Backprop, Guided Grad-CAM の実行 (10s, 60sに限定)
    #------------------------------------------------------------------
    import cv2

    # 対象クラスIDとクラス名
    target_cam_classes = {0:'10s', 5:'60s'}

    # valデータからサンプル画像を取得
    base_val_dir = os.path.join(data_dir, 'val')
    sample_images = {}
    for cls_id, cls_name in target_cam_classes.items():
        cls_dir = os.path.join(base_val_dir, cls_name)
        img_name = os.listdir(cls_dir)[0]
        img_path = os.path.join(cls_dir, img_name)
        sample_images[cls_id] = img_path

    # Grad-CAM用フック設定
    target_layer = model.features[-1]
    features = None
    gradients = None

    def forward_hook(module, input, output):
        nonlocal features
        features = output

    def backward_hook(module, grad_input, grad_output):
        nonlocal gradients
        gradients = grad_output[0]

    forward_h = target_layer.register_forward_hook(forward_hook)
    backward_h = target_layer.register_full_backward_hook(backward_hook)

    # Guided Backprop用：ConvNeXtはGELUを使用, そのためGELUに対するGuided Backpropを実装
    # 出力が正の部分のみ勾配を通す
    gelu_outputs = {}  # moduleをキーにしてforward出力を保存

    def gelu_forward_hook(module, input, output):
        gelu_outputs[module] = output

    def gelu_backward_hook(module, grad_input, grad_output):
        # grad_input: tuple of gradients wrt input of gelu
        # grad_output: tuple of gradients wrt output of gelu
        # guided backprop: 出力が正の位置のみ勾配を通す
        out = gelu_outputs[module]
        # outと同じ形状で, out>0のとこだけ1, それ以外0
        positive_mask = (out > 0).float()
        # grad_output[0]に対して, positive_maskをかけて負の領域をカット
        guided_grad = grad_output[0] * positive_mask
        return (guided_grad,)

    # GELU層にフックを登録
    guided_hooks = []
    for m in model.modules():
        if isinstance(m, nn.GELU):
            fh = m.register_forward_hook(gelu_forward_hook)
            bh = m.register_backward_hook(gelu_backward_hook)
            guided_hooks.append(fh)
            guided_hooks.append(bh)

    def preprocess_image(img_path):
        img = Image.open(img_path).convert('RGB')
        img = img.resize((224, 224))
        img_tensor = data_transforms['val'](img).unsqueeze(0).to(device)
        return img, img_tensor

    def generate_gradcam():
        pooled_gradients = torch.mean(gradients, dim=[0,2,3])
        cam = torch.zeros(features.shape[2:], dtype=features.dtype, device=features.device)
        for i in range(features.shape[1]):
            cam += pooled_gradients[i] * features[0,i,:,:]
        cam = cam.cpu().data.numpy()
        cam = np.maximum(cam, 0)
        if np.max(cam) != 0:
            cam = cam / np.max(cam)
        return cam

    def do_guided_backprop(model, img_tensor, target_class):
        # 勾配リセット
        model.zero_grad()
        img_tensor.grad = None
        # forward
        output = model(img_tensor)
        loss = output[0, target_class]
        model.zero_grad()
        loss.backward()
        guided_grad = img_tensor.grad.data[0].cpu().numpy().transpose(1,2,0)
        guided_grad = guided_grad - guided_grad.min()
        guided_grad = guided_grad / (guided_grad.max() + 1e-8)
        return guided_grad

    def apply_colormap_on_image(org_img, cam, alpha=0.5):
        H_org, W_org, _ = org_img.shape
        cam_resized = cv2.resize(cam, (W_org, H_org))

        heatmap = cv2.applyColorMap(np.uint8(255*cam_resized), cv2.COLORMAP_JET)
        heatmap = np.float32(heatmap)/255.0
        org_img = np.float32(org_img)/255.0

        cam_img = heatmap * alpha + org_img
        cam_img = cam_img / np.max(cam_img)
        return np.uint8(255*cam_img)

    def guided_gradcam(guided_grad, cam):
        H, W, _ = guided_grad.shape
        cam_resized = cv2.resize(cam, (W, H))
        guided_gradcam = guided_grad * cam_resized[..., np.newaxis]
        guided_gradcam = guided_gradcam - guided_gradcam.min()
        guided_gradcam = guided_gradcam / (guided_gradcam.max()+1e-8)
        return guided_gradcam

    gradcam_dir = 'data/result/gradcam_guided'
    if not os.path.exists(gradcam_dir):
        os.makedirs(gradcam_dir)

    model.eval()
    from PIL import Image
    for cls_id, img_path in sample_images.items():
        # original画像読み込み
        org_img, img_tensor = preprocess_image(img_path)
        org_img_np = np.array(org_img)  # RGB, 224x224
        img_tensor.requires_grad = True

        # Grad-CAM
        model.zero_grad()
        output = model(img_tensor)
        target_score = output[0, cls_id]
        target_score.backward()
        cam = generate_gradcam()

        # Grad-CAM画像
        org_img_cv = org_img_np[:,:,::-1].copy()
        gradcam_img = apply_colormap_on_image(org_img_cv, cam)
        gradcam_img_rgb = gradcam_img[:,:,::-1]

        # Guided Backprop
        gb = do_guided_backprop(model, img_tensor, cls_id) # 0-1 float
        g_gradcam = guided_gradcam(gb, cam) # 0-1 float

        original_uint8 = org_img_np
        gradcam_uint8 = gradcam_img_rgb
        gb_uint8 = (gb*255).astype(np.uint8)
        g_gradcam_uint8 = (g_gradcam*255).astype(np.uint8)

        combined = np.hstack([original_uint8, gradcam_uint8, gb_uint8, g_gradcam_uint8])
        plt.imsave(os.path.join(gradcam_dir, f'class_{cls_id}_combined.png'), combined)

    forward_h.remove()
    backward_h.remove()
    for h in guided_hooks:
        h.remove()

    print("Grad-CAM, Guided Backprop, Guided Grad-CAM 完了")
#------------------------------------------------------------------
#スクリプトとして実行された場合(python convnext.py)で実行された場合に,
# if __name__ == '__main__': 以下のみが実行される.
# 並列処理(multiprocessing)を行う場合にこのようにしないと,
# 各処理ですべて同じコードが実行されるため,無限ループなどが起きる.
if __name__ == '__main__':
    main()