Add aarch64-linux target and bump version

.gitignore

@@ -1,4 +1,6 @@
 .stack-work/
+dist-newstyle
 .vscode/
 *.swp
 result
+bench.html

README.md

@@ -6,12 +6,6 @@
 $ nix build
 ```
 
-## Running
-
-```
-$ nix run
-```
-
 ## Building the Docker image
 
 ```
@@ -26,9 +20,7 @@ Includes Stack, GHC, `haskell-language-server`, `gen-hie` etc.
 $ nix develop
 ```
 
-## Running Stack and GHC
+## TODO
 
-```
-$ nix run .#stack
-$ nix run .#ghc
-```
+- [x] Benchmark, profile and optimize search
+- [ ] Trim down the docker image

app/Main.hs

@@ -3,6 +3,7 @@
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE FlexibleInstances #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE OverloadedRecordDot #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RankNTypes #-}
@@ -12,11 +13,13 @@
 
 module Main (main) where
 
+import Control.DeepSeq (force)
+import Control.Exception (evaluate)
+import Control.Monad (void)
 import Data.Char (chr)
 import Data.FileEmbed (embedFile)
 import Data.Foldable (for_)
-import Data.List (intercalate)
-import Data.Maybe (fromMaybe, maybeToList)
+import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Word (Word8)
 import Network.HTTP.Media ((//), (/:))
@@ -48,15 +51,19 @@ import qualified Data.Text.Encoding as Encoding
 import qualified Network.Wai.Handler.Warp as Warp
 import qualified Text.Blaze.Html5 as H
 import qualified Text.Blaze.Html5.Attributes as A
-import qualified Unicode.Char.General.Blocks as UnicodeBlocks
-import qualified Unicode.Char.General.Names as UnicodeNames
 
 import qualified UToy.Decode as Decode
 import qualified UToy.Table as Table
+import qualified UToy.Names as Names
 import qualified UToy.Parsers as Parsers
 
 main :: IO ()
-main = Warp.run 3000 app
+main = do
+  -- Forced evaluation of Names.lowerNames to reduce number of thunks.
+  putStrLn "* Building search index"
+  void $ evaluate $ force Names.lowerNames
+  putStrLn "* Listening on http://localhost:3000"
+  Warp.run 3000 app
 
 app :: Application
 app = serve (Proxy :: Proxy API) server
@@ -100,6 +107,7 @@ examples =
   [ "/text/✅🤔"
   , "/codepoints/x2705+x1F914"
   , "/utf8/e2.9c.85.f0.9f.a4.94"
+  , "/search/asterisk"
   ]
 
 instance MimeRender PlainText RootModel where
@@ -143,8 +151,8 @@ instance MimeRender PlainText Utf8Model where
             Right c ->
               [ Text.pack [c]
               , Text.pack $ printf "U+%04X" c
-              , Text.pack $ intercalate ", " $ allNames c
-              , Text.pack $ fromMaybe "" $ blockName c
+              , Text.intercalate ", " $ Names.allNames c
+              , fromMaybe "" $ Names.blockName c
               ]
           )
         ]
@@ -162,8 +170,8 @@ instance MimeRender HTML Utf8Model where
           Right c -> do
             H.td $ H.input ! A.class_ "charbox" ! A.value (H.toValue [c])
             H.td $ H.code $ printfHtml "U+%04X" c
-            H.td $ H.code $ H.toHtml $ intercalate ", " $ allNames c
-            H.td $ H.code $ H.toHtml $ fromMaybe "" $ blockName c
+            H.td $ H.code $ H.toHtml $ Text.intercalate ", " $ Names.allNames c
+            H.td $ H.code $ H.toHtml $ fromMaybe "" $ Names.blockName c
 
 -- /codepoints/<codepoints>
 
@@ -177,14 +185,14 @@ mkCodepointsModel =
   -- Limit number of returned codepoints. Otherwise it's
   -- too easy to provoke massive response bodies with requests like
   -- /codepoints/0-99999999
-  . take 100000
+  . take 100_000
   . map go
   . concatMap (uncurry enumFromTo)
   where
     go codepoint = (codepoint, toChar codepoint)
 
     toChar codepoint
-      | codepoint > 0x10FFFF = Left "Would be too big (maximum: U+10FFFF)"
+      | codepoint > 0x10_FFFF = Left "Would be too big (maximum: U+10FFFF)"
       | isSurrogate codepoint = Left "Is a surrogate"
       | otherwise = Right $ chr $ fromIntegral codepoint
 
@@ -200,8 +208,8 @@ instance MimeRender PlainText CodepointsModel where
         Right c ->
           [ Text.pack [c]
           , Text.pack $ printf "U+%04X" c
-          , Text.pack $ intercalate ", " $ allNames c
-          , Text.pack $ fromMaybe "" $ blockName c
+          , Text.intercalate ", " $ Names.allNames c
+          , fromMaybe "" $ Names.blockName c
           ]
       )
     | (codepoint, eiC) <- model.codepoints
@@ -218,8 +226,8 @@ instance MimeRender HTML CodepointsModel where
           Right c -> do
             H.td $ H.input ! A.class_ "charbox" ! A.value (H.toValue [c])
             H.td $ H.code $ printfHtml "U+%04X" c
-            H.td $ H.code $ H.toHtml $ intercalate ", " $ allNames c
-            H.td $ H.code $ H.toHtml $ fromMaybe "" $ blockName c
+            H.td $ H.code $ H.toHtml $ Text.intercalate ", " $ Names.allNames c
+            H.td $ H.code $ H.toHtml $ fromMaybe "" $ Names.blockName c
 
 -- /text/<text>
 
@@ -239,10 +247,7 @@ newtype SearchModel = SearchModel
   }
 
 mkSearchModel :: Text -> SearchModel
-mkSearchModel search = SearchModel $ searchAllChars search
-
-searchAllChars :: Text -> [Char]
-searchAllChars search = [c | c <- [minBound..maxBound], any (\name -> Text.toLower search `Text.isInfixOf` Text.toLower (Text.pack name)) (allNames c)]
+mkSearchModel search = SearchModel $ take 100_000 $ Names.searchCaseInsensitive search
 
 instance MimeRender HTML SearchModel where
   mimeRender _ model = charTableHtml model.results
@@ -259,8 +264,8 @@ charTableHtml chars =
       H.tr $ do
         H.td $ H.input ! A.class_ "charbox" ! A.value (H.toValue [c])
         H.td $ H.code $ printfHtml "U+%04X" c
-        H.td $ H.code $ H.toHtml $ intercalate ", " $ allNames c
-        H.td $ H.code $ H.toHtml $ fromMaybe "" $ blockName c
+        H.td $ H.code $ H.toHtml $ Text.intercalate ", " $ Names.allNames c
+        H.td $ H.code $ H.toHtml $ fromMaybe "" $ Names.blockName c
 
 
 charTableText :: [Char] -> BL.ByteString
@@ -269,8 +274,8 @@ charTableText chars =
     [ map Table.cl
       [ Text.pack [c]
       , Text.pack $ printf "U+%04X" c
-      , Text.pack $ intercalate ", " $ allNames c
-      , Text.pack $ fromMaybe "" $ blockName c
+      , Text.intercalate ", " $ Names.allNames c
+      , fromMaybe "" $ Names.blockName c
       ]
     | c <- chars
     ]
@@ -284,15 +289,6 @@ showByteHex = printf "      %02X"
 showByteBin :: Word8 -> String
 showByteBin = printf "%08b"
 
--- | Retrieve name and aliases (suffixed with @*@) of a 'Char'.
-allNames :: Char -> [String]
-allNames c =
-  maybeToList (UnicodeNames.name c)
-  ++ map (++ "*") (UnicodeNames.nameAliases c)
-
-blockName :: Char -> Maybe String
-blockName c = UnicodeBlocks.blockName . UnicodeBlocks.blockDefinition <$> UnicodeBlocks.block c
-
 orThrow :: Either a b -> (a -> ServerError) -> Handler b
 orThrow (Left err) f = throwError $ f err
 orThrow (Right val) _ = pure val

bench/Main.hs

@@ -0,0 +1,69 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Main where
+
+import Criterion.Main (bench, bgroup, defaultMain, whnf)
+import Data.Char (ord)
+import Data.Maybe (maybeToList)
+import Data.Text (Text)
+
+import qualified Data.Text as Text
+import qualified Data.Vector as Vector
+import qualified Unicode.Char.General.Names as UnicodeNames
+
+main = defaultMain
+  [ bench "naiveSearchCI" $ whnf naiveSearchCI "latin"
+  , bench "naiveSearchCS" $ whnf naiveSearchCS "LATIN"
+  , bench "memoSearchCI" $ whnf memoSearchCI "latin"
+  , bench "memoSearchCS" $ whnf memoSearchCS "LATIN"
+  ]
+
+naiveSearchCI :: Text -> [Char]
+naiveSearchCI search =
+  filter go [minBound..maxBound]
+  where
+    go c = any matches $ allNames c
+    matches t = Text.toLower search `Text.isInfixOf` Text.toLower (Text.pack t)
+
+naiveSearchCS :: Text -> [Char]
+naiveSearchCS search =
+  filter go [minBound..maxBound]
+  where
+    go c = any matches $ allNames c
+    matches t = search `Text.isInfixOf` Text.pack t
+
+memoSearchCS :: Text -> [Char]
+memoSearchCS search =
+  filter go [minBound..maxBound]
+  where
+    go c = any matches $ allNamesText c
+    matches t = search `Text.isInfixOf` t
+
+memoSearchCI :: Text -> [Char]
+memoSearchCI search =
+  filter go [minBound..maxBound]
+  where
+    go c = any matches $ allNamesTextLower c
+    matches t = Text.toLower search `Text.isInfixOf` t
+
+-- | Retrieve name and aliases (suffixed with @*@) of a 'Char'.
+allNames :: Char -> [String]
+allNames c =
+  maybeToList (UnicodeNames.name c)
+  ++ map (++ "*") (UnicodeNames.nameAliases c)
+
+allNamesText :: Char -> [Text]
+allNamesText c = Vector.unsafeIndex textNames $ ord c
+
+textNames :: Vector.Vector [Text]
+textNames = Vector.fromList $ map go [minBound..maxBound]
+  where
+    go c = map Text.pack $ allNames c
+
+allNamesTextLower :: Char -> [Text]
+allNamesTextLower c = Vector.unsafeIndex textNamesLower $ ord c
+
+textNamesLower :: Vector.Vector [Text]
+textNamesLower = Vector.fromList $ map go [minBound..maxBound]
+  where
+    go c = map (Text.toLower . Text.pack) $ allNames c

flake.lock

@@ -1,25 +1,59 @@
 {
   "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1703693486,
-        "narHash": "sha256-tuzNTOs+1zR2BEVKKrRRGdpR/n095AXIcT8Me1px2bI=",
+        "lastModified": 1751290243,
+        "narHash": "sha256-kNf+obkpJZWar7HZymXZbW+Rlk3HTEIMlpc6FCNz0Ds=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "671c2d3e1506a7ee1583515ca80cb3474fdc9c95",
+        "rev": "5ab036a8d97cb9476fbe81b09076e6e91d15e1b6",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "release-23.11",
+        "ref": "release-24.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
+        "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -1,68 +1,42 @@
 {
   description = "Unicode toy";
 
-  inputs.nixpkgs.url = "github:nixos/nixpkgs/release-23.11";
+  inputs.nixpkgs.url = "github:nixos/nixpkgs/release-24.11";
+  inputs.flake-utils.url = "github:numtide/flake-utils";
 
-  outputs = { self, nixpkgs }:
+  outputs = { self, nixpkgs, flake-utils }:
+  flake-utils.lib.eachSystem ["x86_64-linux" "aarch64-linux" "aarch64-darwin"] (system:
       let
-      settings = import ./nix/settings.nix;
-      haskellDeps = import ./nix/haskell-deps.nix;
+        pkgs = import nixpkgs { inherit system; };
 
-      pkgs = nixpkgs.legacyPackages.x86_64-linux;
-      haskellPackages = pkgs.haskell.packages."${settings.ghc}";
+        inherit (pkgs.haskell.lib) compose;
 
-      ghc = haskellPackages.ghcWithPackages haskellDeps;
+        # Fix broken unicode-data-names by overriding the version of the unicode-data package.
+        haskellPackagesOverrides = self: super: {
+          unicode-data = super.unicode-data_0_6_0;
+          unicode-data-names = pkgs.haskell.lib.markUnbroken super.unicode-data-names;
+        };
 
-      # Wrap stack to disable its slow Nix integration.
-      # Instead, make it use the GHC defined above.
-      stack = pkgs.stdenv.mkDerivation {
-        name = "stack";
-
-        # The build is simply a call to makeWrapper, so we don't have to
-        # do any of the typical build steps.
-        phases = [ "installPhase" ];
-
-        nativeBuildInputs = [ pkgs.makeWrapper ];
-        # makeBinaryWrapper creates a stack executable for us that uses
-        # the GHC defined in this file.
-        installPhase = ''
-          makeWrapper ${pkgs.stack}/bin/stack $out/bin/stack \
-            --prefix PATH : ${ghc}/bin \
-            --add-flags '--no-nix --system-ghc --no-install-ghc'
+        utoy = pkgs.lib.pipe
+          # The basic package
+          (pkgs.haskellPackages.developPackage {
+            root = ./.;
+            overrides = haskellPackagesOverrides;
+          })
+          # And some build configuration.
+          # See https://nixos.org/manual/nixpkgs/unstable/#haskell-packaging-helpers.
+          [
+            # Remove warp which pulls GHC into the runtime deps for some reason.
+            (compose.overrideCabal (drv: {
+              postInstall = ''
+                remove-references-to -t ${pkgs.haskellPackages.warp} $out/bin/utoy
               '';
-      };
-
-      utoy = pkgs.haskell.lib.justStaticExecutables (haskellPackages.callPackage
-        ({ mkDerivation }:
-        mkDerivation {
-          # Keep this in sync with package.yaml
-          version = "0.6.1";
-          pname = "utoy";
-          license = pkgs.lib.licenses.mit;
-          src =
-            # We only need these files for building:
-            let
-              whitelist = [
-                ./LICENSE
-                ./utoy.cabal
-                ./Setup.hs
-                ./app
-                ./src
-                ./static
-                ./test
+            }))
+            # Return only the bin folder to curb image size
+            compose.justStaticExecutables
           ];
-            in
-              pkgs.lib.sources.cleanSourceWith {
-                src = ./.;
-                filter = path: _type: pkgs.lib.any (prefix: pkgs.lib.hasPrefix (toString prefix) path) whitelist;
-              };
-          libraryHaskellDepends = haskellDeps haskellPackages;
-        }) {});
       in {
-      packages.x86_64-linux = {
-        inherit ghc;
-        inherit stack;
-
+        packages = rec {
           docker =
             pkgs.dockerTools.buildImage {
               name = "git.pbrinkmeier.de/paul/utoy";
@@ -73,17 +47,17 @@
           default = utoy;
         };
 
-      devShells.x86_64-linux.default = pkgs.mkShell {
-        packages = [
-          stack
-          ghc
-
-          haskellPackages.haskell-language-server
-          haskellPackages.implicit-hie
+        devShells.default =
+          (pkgs.haskellPackages.developPackage {
+            root = ./.;
+            overrides = haskellPackagesOverrides;
+            modifier = drv:
+              pkgs.haskell.lib.addBuildTools drv [
+                pkgs.cabal-install
+                pkgs.haskellPackages.implicit-hie
+                pkgs.haskell-language-server
               ];
-        shellHook = ''
-          PS1+="(utoy) ";
-        '';
-      };
-  };
+            cabal2nixOptions = "--benchmark";
+          }).env;
+        });
 }

hie.yaml

@@ -1,10 +1,10 @@
 cradle:
-  stack:
-    - path: "./src"
-      component: "utoy:lib"
+  cabal:
+    - path: "src"
+      component: "lib:utoy"
 
-    - path: "./app/Main.hs"
+    - path: "app/Main.hs"
       component: "utoy:exe:utoy"
 
-    - path: "./test"
-      component: "utoy:test:utoy-test"
+    - path: "bench/Main.hs"
+      component: "utoy:bench:utoy-bench"

nix/haskell-deps.nix

@@ -1,13 +0,0 @@
-haskellPackages: with haskellPackages; [
-  attoparsec
-  blaze-html
-  bytestring
-  file-embed
-  http-media
-  servant-server
-  text
-  unicode-data
-  unicode-data-names
-  wai
-  warp
-]

nix/settings.nix

@@ -1,5 +0,0 @@
-{
-  # GHC version to use with Nix.
-  # Should match the one in stack.yaml.
-  ghc = "ghc944";
-}

package.yaml

@@ -1,70 +0,0 @@
-# Adapted from new-template.hsfiles
-
-name:                utoy
-# Keep this in sync with the version in flake.nix.
-version:             0.6.1
-git:                 "https://git.pbrinkmeier.de/paul/utoy"
-license:             MIT
-author:              "Paul Brinkmeier"
-maintainer:          "hallo@pbrinkmeier.de"
-copyright:           "2023 Paul Brinkmeier"
-
-extra-source-files:
-- README.md
-- static/utoy.css
-
-dependencies:
-- base >= 4.7 && < 5
-- attoparsec
-- text
-
-ghc-options:
-- -Wall
-- -Wcompat
-- -Widentities
-- -Wincomplete-record-updates
-- -Wincomplete-uni-patterns
-- -Wmissing-export-lists
-- -Wmissing-home-modules
-- -Wpartial-fields
-- -Wredundant-constraints
-
-library:
-  source-dirs: src
-
-executables:
-  utoy:
-    main:                Main.hs
-    source-dirs:         app
-    ghc-options:
-    - -threaded
-    - -rtsopts
-    - -with-rtsopts=-N
-    dependencies:
-    - utoy
-    - blaze-html
-    - bytestring
-    - file-embed
-    - http-media
-    - servant-server
-    - text
-    - unicode-data
-    - unicode-data-names
-    - wai
-    - warp
-    # Fix "Multiple files use the same module name", see
-    # https://stackoverflow.com/questions/67519851/multiple-files-use-the-same-module-name
-    when:
-    - condition: false
-      other-modules: Paths_utoy
-
-tests:
-  utoy-test:
-    main:                Spec.hs
-    source-dirs:         test
-    ghc-options:
-    - -threaded
-    - -rtsopts
-    - -with-rtsopts=-N
-    dependencies:
-    - utoy

src/UToy/Names.hs

@@ -0,0 +1,31 @@
+module UToy.Names (allNames, blockName, lowerNames, searchCaseInsensitive) where
+
+import Data.Char (ord)
+import Data.Maybe (maybeToList)
+import Data.Text (Text)
+import Data.Vector (Vector)
+
+import qualified Data.Text as Text
+import qualified Data.Vector as Vector
+import qualified Unicode.Char.General.Blocks as UnicodeBlocks
+import qualified Unicode.Char.General.Names as UnicodeNames
+
+-- | Retrieve name and aliases (suffixed with @*@) of a 'Char'.
+allNames :: Char -> [Text]
+allNames c = map Text.pack $
+  maybeToList (UnicodeNames.name c)
+  ++ map (++ "*") (UnicodeNames.nameAliases c)
+
+blockName :: Char -> Maybe Text
+blockName c = Text.pack . UnicodeBlocks.blockName . UnicodeBlocks.blockDefinition <$> UnicodeBlocks.block c
+
+searchCaseInsensitive :: Text -> [Char]
+searchCaseInsensitive search = filter go [minBound..maxBound]
+  where
+    go c = any matches $ Vector.unsafeIndex lowerNames (ord c - ord minBound)
+    matches t = Text.toLower search `Text.isInfixOf` t
+
+lowerNames :: Vector (Vector Text)
+lowerNames = Vector.fromList $ map go [minBound..maxBound]
+  where
+    go = Vector.fromList . map Text.toLower . allNames

stack.yaml

@@ -1,5 +0,0 @@
-# You can get a working environment using nix develop.
-# Keep this in sync with nix/settings.nix
-resolver: ghc-9.4.4
-packages:
-- .

test/Spec.hs

@@ -1,2 +0,0 @@
-main :: IO ()
-main = putStrLn "Test suite not yet implemented"

utoy.cabal

@@ -1,11 +1,7 @@
 cabal-version: 1.12
 
--- This file has been generated from package.yaml by hpack version 0.35.2.
---
--- see: https://github.com/sol/hpack
-
 name:           utoy
-version:        0.6.1
+version:        0.6.3
 author:         Paul Brinkmeier
 maintainer:     hallo@pbrinkmeier.de
 copyright:      2023 Paul Brinkmeier
@@ -24,6 +20,7 @@ library
   exposed-modules:
       UToy.Decode
       UToy.Parsers
+      UToy.Names
       UToy.Table
   other-modules:
       Paths_utoy
@@ -34,6 +31,9 @@ library
       attoparsec
     , base >=4.7 && <5
     , text
+    , unicode-data
+    , unicode-data-names
+    , vector
   default-language: Haskell2010
 
 executable utoy
@@ -46,28 +46,25 @@ executable utoy
     , base >=4.7 && <5
     , blaze-html
     , bytestring
+    , deepseq
     , file-embed
     , http-media
     , servant-server
     , text
-    , unicode-data
-    , unicode-data-names
     , utoy
     , wai
     , warp
   default-language: Haskell2010
 
-test-suite utoy-test
+benchmark utoy-bench
   type: exitcode-stdio-1.0
-  main-is: Spec.hs
-  other-modules:
-      Paths_utoy
+  main-is: Main.hs
   hs-source-dirs:
-      test
-  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N
+    bench
   build-depends:
-      attoparsec
-    , base >=4.7 && <5
+      base >=4.7 && <5
+    , criterion
     , text
-    , utoy
+    , unicode-data-names
+    , vector
   default-language: Haskell2010