Машинное обучение для анализа тепловых карт и оптимизации конверсии страниц

import pandas as pd

# 1. Load raw data
df = pd.read_csv('raw/heatmap_raw.csv')

# 2. Deduplicate
df = df.drop_duplicates(subset=['session_id', 'timestamp'])

# 3. Filter short sessions and bots
bot_patterns = ['bot', 'crawl', 'spider']
df = df[~df['user_agent'].str.contains('|'.join(bot_patterns), case=False, na=False)]
df['duration'] = df.groupby('session_id')['timestamp'].transform('last') - \
                 df.groupby('session_id')['timestamp'].transform('first')
df = df[df['duration'] >= 5000]

# 4. Normalize coordinates
df['x_norm'] = df['x'] / df['viewport_w']
df['y_norm'] = df['y'] / df['viewport_h']

# 5. Annotate event types
def annotate(ev):
    if ev == 'click': return 'click'
    if ev == 'scroll': return 'scroll'
    if ev == 'hover': return 'hover'
    return 'exit'
df['event_type'] = df['event'].apply(annotate)

# 6. Compute session metrics
metrics = df.groupby('session_id').agg(
    dwell_time=('timestamp', lambda x: x.max() - x.min()),
    scroll_depth=('y_norm', 'max'),
    heat_density=('event', 'count')
).reset_index()
metrics['scroll_depth'] *= 100

# 7. Merge metrics back
df = df.merge(metrics, on='session_id', how='left')

# 8. Final quality check
print(df.shape)
print(df.isna().sum())
print(df['event_type'].value_counts())

# 9. Save
df.to_parquet('data/heatmap_clean.parquet', index=False)

import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader, random_split
from torchvision import transforms, models

# 1. Dataset
class HeatmapDataset(Dataset):
    def __init__(self, images, labels, transform=None):
        self.images = images
        self.labels = labels
        self.transform = transform
    def __len__(self):
        return len(self.images)
    def __getitem__(self, idx):
        img = self.images[idx]
        lbl = self.labels[idx]
        if self.transform:
            img = self.transform(img)
        return img, lbl

# 2. Preprocessing
transform = transforms.Compose([
    transforms.ToTensor(),                 # (H, W) → (C, H, W)
    transforms.Normalize([0.5], [0.5])      # (0,1) → (-1,1)
])

# 3. Load data (replace with actual loader)
images = [...]   # list/array of numpy arrays
labels = [...]   # list/array of 0/1
dataset = HeatmapDataset(images, labels, transform=transform)

# 4. Train/val split
train_size = int(0.8 * len(dataset))
val_size = len(dataset) - train_size
train_ds, val_ds = random_split(dataset, [train_size, val_size])

train_loader = DataLoader(train_ds, batch_size=32, shuffle=True, num_workers=4)
val_loader   = DataLoader(val_ds, batch_size=32, shuffle=False, num_workers=4)

# 5. Model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = models.resnet18(pretrained=False)
model.fc = nn.Linear(model.fc.in_features, 1)
model = model.to(device)

# 6. Loss & optimizer
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',
                                                      patience=3, factor=0.5)

# 7. Training loop
best_val_loss = float('inf')
for epoch in range(10):
    # Train
    model.train()
    train_loss = 0.0
    for imgs, lbls in train_loader:
        imgs, lbls = imgs.to(device), lbls.to(device).float()
        optimizer.zero_grad()
        outputs = model(imgs).squeeze()
        loss = criterion(outputs, lbls)
        loss.backward()
        optimizer.step()
        train_loss += loss.item() * imgs.size(0)
    train_loss /= len(train_loader.dataset)

    # Validate
    model.eval()
    val_loss = 0.0
    with torch.no_grad():
        for imgs, lbls in val_loader:
            imgs, lbls = imgs.to(device), lbls.to(device).float()
            outputs = model(imgs).squeeze()
            loss = criterion(outputs, lbls)
            val_loss += loss.item() * imgs.size(0)
    val_loss /= len(val_loader.dataset)

    scheduler.step(val_loss)

    print(f'Epoch {epoch+1:02d} | Train loss: {train_loss:.4f} | Val loss: {val_loss:.4f}')

    # Save best
    if val_loss