Terrible Search Engine

Using simulated annealing on websites with links pointing to neighbours is not a good idea, but a good example for async annealing.

from urllib.parse import urljoin
from random import choice

from anyio import sleep
from httpx import AsyncClient
from bs4 import BeautifulSoup
from altbacken.core.async_annealing import AnnealingState
from altbacken.external.async_annealing import SimpleAsyncSimulatedAnnealing

type URL = str | None
type Weight = int

SEARCH_KEYS: dict[str, Weight] = {
    "Python": -1,
    "Moon": -5,
    "Bass": -8,
    "Cheesecake": -10
}

class LinkNeighbourhood:
    async def __call__(self, state: AnnealingState[URL]) -> URL:
        if current_url := state.current_solution:
            try:
                async with AsyncClient(follow_redirects=True) as client:
                    response = await client.get(current_url)
                    if not response.is_success:
                        return current_url
                    await sleep(0.5)
                    soup = BeautifulSoup(response.text, "html.parser")
                    links: list[str] = [
                        urljoin(current_url, link["href"])
                        for link in soup.find_all("a", {"href": True})
                    ]
                    return choice(links) if links else None
            except Exception:
                return current_url
        else:
            return None

async def fitness(url: URL) -> float:
    if url is None:
        return 100.0
    async with AsyncClient(follow_redirects=True) as client:
        response = await client.get(url)
        await sleep(0.5)
        if not response.is_success:
            return 50.0
        text: str = response.text
        return sum(
            text.count(key) * weight
            for key, weight in SEARCH_KEYS.items()
        )

annealing: SimpleAsyncSimulatedAnnealing[URL] = SimpleAsyncSimulatedAnnealing(
    fitness,
    LinkNeighbourhood()
)

visited_sites: set[URL] = set()

async def tracer(state: AnnealingState[URL]) -> None:
    visited_sites.add(state.current_solution)

annealing.tracer = tracer

best_url, value = await annealing.simulate("https://stackoverflow.com/questions")

best_url, value

('https://stackoverflow.com/questions/79972182/scipy-import-from-module-fails-sometimes-path-dependent',
 -6)

visited_sites

{None,
 'https://biology.stackexchange.com/users/129209/keiv',
 'https://stackexchange.com',
 'https://stackoverflow.com/questions/79972182/scipy-import-from-module-fails-sometimes-path-dependent',
 'https://stackoverflowteams.com',
 'https://stackoverflowteams.com/teams/create/free/?utm_medium=referral&utm_source=biology-community&utm_campaign=side-bar&utm_content=explore-teams',
 'https://stackoverflowteams.com/teams/create/free?isLogin=false',
 'https://stackoverflowteams.com/teams/create/free?isLogin=true',
 'https://www.instagram.com/thestackoverflow'}