{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Hbol",
   "metadata": {},
   "outputs": [],
   "source": [
    "from json import dumps as json_dumps\n",
    "\n",
    "import marimo as mo\n",
    "import plotly.express as px\n",
    "import polars as pl\n",
    "from bs4 import BeautifulSoup\n",
    "from bs4.element import Tag\n",
    "from httpx import get as http_get\n",
    "from pydantic import BaseModel\n",
    "from pydantic_ai.agent import Agent\n",
    "from pydantic_ai.models.openai import OpenAIModel\n",
    "from pydantic_ai.providers.openai import OpenAIProvider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "MJUe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.offline\n",
    "plotly.offline.init_notebook_mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "vblA",
   "metadata": {},
   "outputs": [],
   "source": [
    "_SHAARLI_URL = \"https://sebsauvage.net/links/\"\n",
    "\n",
    "class Link(BaseModel):\n",
    "    id: str\n",
    "    title: str\n",
    "    href: str\n",
    "    description: str\n",
    "    tags: list[str]\n",
    "\n",
    "def extract_link(entry_node: Tag) -> Link:\n",
    "    link_id = entry_node.select_one(\"a\")[\"id\"]\n",
    "\n",
    "    container_node = entry_node.select_one(\".linkcontainer\")\n",
    "\n",
    "    link_node = container_node.select_one(\".linktitle a\")\n",
    "    link_href = link_node[\"href\"]\n",
    "    if link_href == \"?\" + link_id:\n",
    "        link_href = \"\"\n",
    "    link_title = link_node.text.strip()\n",
    "\n",
    "    if description_node := container_node.select_one(\".linkdescription\"):\n",
    "        link_description = description_node.text.strip()\n",
    "    else:\n",
    "        link_description = \"\"\n",
    "\n",
    "    link_tags = [\n",
    "        tag.text.strip() for tag in container_node.select(\".linktaglist a\")\n",
    "    ]\n",
    "\n",
    "    link = Link(\n",
    "        id=link_id,\n",
    "        title=link_title,\n",
    "        href=link_href,\n",
    "        description=link_description,\n",
    "        tags=link_tags,\n",
    "    )\n",
    "\n",
    "    return link\n",
    "\n",
    "def extract_links(soup: BeautifulSoup) -> list[Link]:\n",
    "    return [extract_link(entry_node) for entry_node in soup.select(\"#linklist li\")]\n",
    "\n",
    "_r = http_get(_SHAARLI_URL)\n",
    "_r.raise_for_status()\n",
    "\n",
    "_soup = BeautifulSoup(_r.text, \"html5lib\")\n",
    "_links = extract_links(_soup)\n",
    "links_df = pl.DataFrame(_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bkHC",
   "metadata": {},
   "outputs": [],
   "source": [
    "links_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lEQa",
   "metadata": {},
   "outputs": [],
   "source": [
    "_OLLAMA_URL = \"http://edemaruh:11434/v1\"\n",
    "_LLM_MODEL = \"ollama:qwen3:8b\"\n",
    "# LLM_MODEL = \"google-gla:gemini-2.5-flash\"\n",
    "\n",
    "_SYSTEM_PROMPT = \"\"\"\\\n",
    "You are a data classifier. for each content item, determines whether it talks about AI/LLM (True) or not (False). \n",
    "\"\"\"\n",
    "\n",
    "_provider_name, _model_name = _LLM_MODEL.split(\":\", maxsplit=1)\n",
    "if _provider_name == \"ollama\":\n",
    "    _provider = OpenAIProvider(\n",
    "        base_url=_OLLAMA_URL,\n",
    "    )\n",
    "    _model = OpenAIModel(\n",
    "        model_name=_model_name,\n",
    "        provider=_provider,\n",
    "    )\n",
    "else:\n",
    "    _model = _LLM_MODEL\n",
    "\n",
    "_agent = Agent(model=_model, output_type=bool, system_prompt=_SYSTEM_PROMPT)\n",
    "\n",
    "classified_links_df = links_df.with_columns(\n",
    "    pl.Series(\n",
    "        \"is_ai\",\n",
    "        [\n",
    "            (await _agent.run(json_dumps(link))).output\n",
    "            for link in mo.status.progress_bar(\n",
    "                links_df.iter_rows(), total=links_df.height\n",
    "            )\n",
    "        ],\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "PKri",
   "metadata": {},
   "outputs": [],
   "source": [
    "classified_links_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Xref",
   "metadata": {},
   "outputs": [],
   "source": [
    "px.pie(\n",
    "    classified_links_df,\n",
    "    names=\"is_ai\",\n",
    "    title=\"AI/LLM Links Classification\",\n",
    "    color_discrete_sequence=px.colors.qualitative.Pastel,\n",
    "    hole=0.3,\n",
    ").show()"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}