AI‑аудит E‑AT: как машин обучение проверяет экспертность авторитетность надёжность

import json
import os
from google.oauth2 import service_account
from googleapiclient.discovery import build

# 1. Инициализация API
SCOPES = ['https://www.googleapis.com/auth/webmasters.readonly']
KEY_PATH = os.getenv('GSC_KEY')
creds = service_account.Credentials.from_service_account_file(KEY_PATH, scopes=SCOPES)
service = build('webmasters', 'v3', credentials=creds)

def run_eat_audit(data):
    # вызов модели (пример)
    result = ai_model.analyze(data)
    return result

def push_to_gsc(report):
    # отправка ключевых метрик в GSC
    body = {
        'reportType': 'EAT',
        'reportData': report
    }
    service.searchanalytics().query(siteUrl='https://example.com', body=body).execute()

# 2. Основной поток
if __name__ == "__main__":
    raw_data = load_pipeline_output()          # ваш существующий пайплайн
    audit = run_eat_audit(raw_data)
    with open('audits/report.json', 'w') as f:
        json.dump(audit, f, indent=2)
    push_to_gsc(audit)

# eat_metrics.py
import requests
from bs4 import BeautifulSoup
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Ridge
import json
import re

# ---------- 1. Скачиваем страницу ----------
def fetch_html(url: str) -> str:
    headers = {"User-Agent": "Mozilla/5.0 (compatible; EAT-Scanner/1.0)"}
    resp = requests.get(url, headers=headers, timeout=10)
    resp.raise_for_status()
    return resp.text

# ---------- 2. Извлекаем метаданные ----------
def extract_metadata(soup: BeautifulSoup) -> dict:
    meta = {}
    meta['title'] = soup.title.string.strip() if soup.title else ""
    meta['description'] = (
        soup.find("meta", attrs={"name": "description"}) or
        soup.find("meta", attrs={"property": "og:description"})
    )
    meta['description'] = meta['description']['content'].strip() if meta['description'] else ""
    # JSON‑LD
    json_ld = []
    for script in soup.find_all("script", type="application/ld+json"):
        try:
            json_ld.append(json.loads(script.string))
        except json.JSONDecodeError:
            continue
    meta['json_ld'] = json_ld
    return meta

# ---------- 3. Извлекаем основной текст ----------
def extract_text(soup: BeautifulSoup) -> str:
    # Убираем скрипты и стили
    for elem in soup(["script", "style", "noscript"]):
        elem.decompose()
    # Берём только тело страницы
    body = soup.body
    if not body:
        return ""
    # Сохраняем текст с разделением по абзацам
    paragraphs = body.find_all(["p", "li"])
    text = "\n".join(p.get_text(strip=True) for p in paragraphs)
    # Убираем лишние пробелы
    text = re.sub(r"\s{2,}", " ", text)
    return text.strip()

# ---------- 4. Подготовка фичей ----------
def build_features(urls: list) -> pd.DataFrame:
    rows = []
    for url in urls:
        html = fetch_html(url)
        soup = BeautifulSoup(html, "html.parser")
        meta = extract_metadata(soup)
        content = extract_text(soup)
        rows.append({
            "url": url,
            "title": meta["title"],
            "description": meta["description"],
            "content": content,
            "json_ld_count": len(meta["json_ld"])
        })
    df = pd.DataFrame(rows)
    return df

# ---------- 5. Векторизация текста ----------
def vectorize(df: pd.DataFrame) -> pd.DataFrame:
    tfidf = TfidfVectorizer(stop_words="english", max_features=500)
    X_text = tfidf.fit_transform(df["content"])
    # Добавляем числовые признаки
    numeric = df[["json_ld_count"]].values
    # Конкатенируем
    from scipy.sparse import hstack
    X = hstack([X_text, numeric])
    return X, tfidf

# ---------- 6. Обучаем модель (пример) ----------
def train_model(X, y):
    scaler = StandardScaler(with_mean=False)
    ridge = Ridge(alpha=1.0)
    pipeline = Pipeline([
        ("scaler", scaler),
        ("model", ridge)
    ])
    pipeline.fit(X, y)
    return pipeline

# ---------- 7. Запуск ----------
if __name__ == "__main__":
    # Пример списка URL‑ов
    urls = [
        "https://example.com/article1",
        "https://example.com/article2",
        # Добавьте свои
    ]
    df = build_features(urls)
    X, tfidf = vectorize(df)
    # Для обучения нужны метки E‑AT (0‑10). Здесь placeholder.
    y = [7, 5]  # Замените реальными данными
    model = train_model(X, y)
    # Сохраняем признаки
    df.to_csv("features.csv", index=False)
    # Сохраняем модель
    import joblib
    joblib.dump(model, "eat_model.pkl")
    print("E‑AT pipeline готов.")