@Test
    fun `can get total pages`() {
        runBlocking {
            withContext(<http://Dispatchers.IO|Dispatchers.IO>) {
                val totalPages = skrape(AsyncFetcher) {
                    request {
                        url = "<https://www.capfriendly.com/browse/active>"
                    }
                    response {
                        htmlDocument {
                            div {
                                withClass = "pagination"
                                findFirst {
                                    div {
                                        withClass = "r"
                                        val paginationText = findByIndex(1).text
                                        paginationText.substringAfter(" of ").toInt()
                                    }
                                }
                            }
                        }
                    }
                }

                println(totalPages)
            }
        }
    }

@Test
    fun `can get total pages`() = runBlocking {
        withContext(<http://Dispatchers.IO|Dispatchers.IO>) {
            val totalPages = skrape(AsyncFetcher) {
                request {
                    url = "<https://www.capfriendly.com/browse/active>"
                }
                response {
                    htmlDocument {
                        div {
                            withClass = "pagination"
                            findFirst {
                                div {
                                    withClass = "r"
                                    a {
                                        findAll { find { it.text == "Last" }?.attribute("data-val") }
                                    }
                                }
                            }
                        }
                    }
                }
            }

            println(totalPages)
        }
    }

private suspend fun fetchImdb(): List<User> =
    withContext(<http://Dispatchers.IO|Dispatchers.IO>) {
        skrape(AsyncFetcher) {
            request {
                url = "<https://www.imdb.com/chart/top/>"
                sslRelaxed = true
            }.also { println("call ${it.preparedRequest.url}") }
            response {
                htmlDocument {
                    table {
                        tbody {
                            withClass = "lister-list"
                            tr {
                                findAll {
                                    map {
                                        val title = <http://it.td|it.td> { findSecond { text } }
                                        User(name = title, "", "")
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

private suspend fun fetchDB(): List<User> {
    var players = listOf<User>()
    withContext(<http://Dispatchers.IO|Dispatchers.IO>) {
        val deferred = ('a'..'z').filterNot { it == 'x' }.map { async { getHockeyDb(it) } }
        players = deferred.awaitAll().flatten()
    }
    println("players total: ${players.size}")
    println("players 5: ${players[5]}")
    println("players 100: ${players[100]}")
    println("players 250: ${players[250]}")
    println("players 500: ${players[500]}")
    println("players 5000: ${players[5000]}")
    return players
}

private suspend fun getHockeyDb(letter: Char): List<User> {
    return withContext(<http://Dispatchers.IO|Dispatchers.IO>) {
        skrape(AsyncFetcher) {
            request {
                url = "<https://www.hockeydb.com/ihdb/players/player_ind_$letter.html>"
                sslRelaxed = true
            }.also { println("call ${it.preparedRequest.url}") }
            response {
                htmlDocument {
                    table {
                        tbody {
                            tr {
                                findAll {
                                    map {
                                        val name = <http://it.td|it.td> { it.a { findFirst { text } } }
                                        val team = <http://it.td|it.td> { findByIndex(1) { text } }
                                        val salary = <http://it.td|it.td> { findLast { text } }
                                        User(name, team, salary)
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}

AsyncFetcher

ktor-client-okhttp